{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to DataLad downloaders\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os.path as op\n\nfrom .base import Interface\nfrom ..core.local.save import Save\nfrom ..interface.base import build_doc\nfrom ..interface.common_opts import nosave_opt\nfrom ..interface.common_opts import save_message_opt\nfrom ..interface.results import get_status_dict\nfrom ..interface.utils import eval_results\nfrom ..utils import assure_list_from_str\nfrom ..distribution.dataset import Dataset\nfrom ..distribution.dataset import datasetmethod\nfrom ..distribution.dataset import EnsureDataset\nfrom ..distribution.dataset import path_under_rev_dataset\nfrom ..distribution.dataset import require_dataset\nfrom ..distribution.dataset import resolve_path\nfrom ..dochelpers import exc_str\nfrom ..support.annexrepo import AnnexRepo, AnnexBatchCommandError\nfrom ..support.param import Parameter\nfrom ..support.constraints import EnsureStr, EnsureNone\nfrom ..support.exceptions import NoDatasetArgumentFound\n\nfrom logging import getLogger\nlgr = getLogger('datalad.api.download-url')\n\n\n@build_doc\nclass DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            metavar='PATH',\n            doc=\"\"\"specify the dataset to add files to. If no dataset is given,\n            an attempt is made to identify the dataset based on the current\n            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to\n            prevent adding files to the dataset.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone()),\n        archive=Parameter(\n            args=(\"--archive\",),\n            action=\"store_true\",\n            doc=\"\"\"pass the downloaded files to [CMD: :command:`datalad\n            add-archive-content --delete` CMD][PY: add_archive_content(...,\n            delete=True) PY]\"\"\"),\n        save=nosave_opt,\n        message=save_message_opt\n    )\n\n    @staticmethod\n    @datasetmethod(name=\"download_url\")\n    @eval_results\n    def __call__(urls, dataset=None, path=None, overwrite=False,\n                 archive=False, save=True, message=None):\n        from ..downloaders.providers import Providers\n\n        ds = None\n        if save or dataset:\n            try:\n                ds = require_dataset(\n                    dataset, check_installed=True,\n                    purpose='downloading urls')\n            except NoDatasetArgumentFound:\n                pass\n\n        common_report = {\"action\": \"download_url\",\n                         \"ds\": ds}\n\n        got_ds_instance = isinstance(dataset, Dataset)\n        path = str(resolve_path(path or op.curdir, ds=dataset))\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not op.isdir(path):\n            yield get_status_dict(\n                status=\"error\",\n                message=(\n                    \"When specifying multiple urls, --path should point to \"\n                    \"an existing directory. Got %r\", path),\n                type=\"file\",\n                path=path,\n                **common_report)\n            return\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        path_urls = {}\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                path_urls[downloaded_path] = url\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n\n        if downloaded_paths and save and ds is not None:\n            msg = message or \"\"\"\\\n[DATALAD] Download URLs\n\nURLs:\n  {}\"\"\".format(\"\\n  \".join(urls))\n\n            for r in Save()(downloaded_paths, message=msg,\n                            # ATTN: Pass the original dataset argument to\n                            # preserve relative path handling semantics.\n                            dataset=dataset,\n                            return_type=\"generator\",\n                            result_xfm=None,\n                            result_filter=None,\n                            on_failure=\"ignore\"):\n                yield r\n\n            if isinstance(ds.repo, AnnexRepo):\n                if got_ds_instance:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # dataset.\n                    rpaths = dict(zip(downloaded_paths, downloaded_paths))\n                else:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # current working directory. Take these relative to the\n                    # dataset for use with the AnnexRepo method calls.\n                    rpaths = {}\n                    for orig_path, resolved in zip(\n                            downloaded_paths,\n                            resolve_path(downloaded_paths, ds=dataset)):\n                        rpath = path_under_rev_dataset(ds, resolved)\n                        if rpath:\n                            rpaths[str(rpath)] = orig_path\n                        else:\n                            lgr.warning(\"Path %s not under dataset %s\",\n                                        orig_path, ds)\n                annex_paths = [p for p, annexed in\n                               zip(rpaths,\n                                   ds.repo.is_under_annex(list(rpaths.keys())))\n                               if annexed]\n                if annex_paths:\n                    for path in annex_paths:\n                        url = path_urls[rpaths[path]]\n                        try:\n                            # The file is already present. This is just to\n                            # register the URL.\n                            ds.repo.add_url_to_file(path, url, batch=True)\n                        except AnnexBatchCommandError as exc:\n                            lgr.warning(\"Registering %s with %s failed: %s\",\n                                        path, url, exc_str(exc))\n\n                    if archive:\n                        from datalad.api import add_archive_content\n                        for path in annex_paths:\n                            add_archive_content(path, annex=ds.repo, delete=True)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to DataLad downloaders\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os.path as op\n\nfrom .base import Interface\nfrom ..core.local.save import Save\nfrom ..interface.base import build_doc\nfrom ..interface.common_opts import nosave_opt\nfrom ..interface.common_opts import save_message_opt\nfrom ..interface.results import get_status_dict\nfrom ..interface.utils import eval_results\nfrom ..utils import assure_list_from_str\nfrom ..distribution.dataset import Dataset\nfrom ..distribution.dataset import datasetmethod\nfrom ..distribution.dataset import EnsureDataset\nfrom ..distribution.dataset import path_under_rev_dataset\nfrom ..distribution.dataset import require_dataset\nfrom ..distribution.dataset import resolve_path\nfrom ..dochelpers import exc_str\nfrom ..support.annexrepo import AnnexRepo, AnnexBatchCommandError\nfrom ..support.param import Parameter\nfrom ..support.constraints import EnsureStr, EnsureNone\nfrom ..support.exceptions import NoDatasetArgumentFound\n\nfrom logging import getLogger\nlgr = getLogger('datalad.api.download-url')\n\n\n@build_doc\nclass DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            metavar='PATH',\n            doc=\"\"\"specify the dataset to add files to. If no dataset is given,\n            an attempt is made to identify the dataset based on the current\n            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to\n            prevent adding files to the dataset.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"\"\"target for download. If the path has a trailing separator,\n            it is treated as a directory, and each specified URL is downloaded\n            under that directory to a base name taken from the URL. Without a\n            trailing separator, the value specifies the name of the downloaded\n            file and only a single URL should be given. In both cases, leading\n            directories will be created if needed. This argument defaults to\n            the current directory.\"\"\",\n            constraints=EnsureStr() | EnsureNone()),\n        archive=Parameter(\n            args=(\"--archive\",),\n            action=\"store_true\",\n            doc=\"\"\"pass the downloaded files to [CMD: :command:`datalad\n            add-archive-content --delete` CMD][PY: add_archive_content(...,\n            delete=True) PY]\"\"\"),\n        save=nosave_opt,\n        message=save_message_opt\n    )\n\n    @staticmethod\n    @datasetmethod(name=\"download_url\")\n    @eval_results\n    def __call__(urls, dataset=None, path=None, overwrite=False,\n                 archive=False, save=True, message=None):\n        from ..downloaders.providers import Providers\n\n        ds = None\n        if save or dataset:\n            try:\n                ds = require_dataset(\n                    dataset, check_installed=True,\n                    purpose='downloading urls')\n            except NoDatasetArgumentFound:\n                pass\n\n        common_report = {\"action\": \"download_url\",\n                         \"ds\": ds}\n\n        got_ds_instance = isinstance(dataset, Dataset)\n        dir_is_target = not path or path.endswith(op.sep)\n        path = str(resolve_path(path or op.curdir, ds=dataset))\n        if dir_is_target:\n            # resolve_path() doesn't preserve trailing separators. Add one for\n            # the download() call.\n            path = path + op.sep\n        urls = assure_list_from_str(urls)\n\n        if not dir_is_target:\n            if len(urls) > 1:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=(\n                        \"When specifying multiple urls, --path should point to \"\n                        \"a directory target (with a trailing separator). Got %r\",\n                        path),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n                return\n            # download() would be fine getting an existing directory and\n            # downloading the URL underneath it, but let's enforce a trailing\n            # slash here for consistency.\n            if op.isdir(path):\n                yield get_status_dict(\n                    status=\"error\",\n                    message=(\n                        \"Non-directory path given (no trailing separator) \"\n                        \"but a directory with that name exists\"),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n                return\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        path_urls = {}\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                path_urls[downloaded_path] = url\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n\n        if downloaded_paths and save and ds is not None:\n            msg = message or \"\"\"\\\n[DATALAD] Download URLs\n\nURLs:\n  {}\"\"\".format(\"\\n  \".join(urls))\n\n            for r in Save()(downloaded_paths, message=msg,\n                            # ATTN: Pass the original dataset argument to\n                            # preserve relative path handling semantics.\n                            dataset=dataset,\n                            return_type=\"generator\",\n                            result_xfm=None,\n                            result_filter=None,\n                            on_failure=\"ignore\"):\n                yield r\n\n            if isinstance(ds.repo, AnnexRepo):\n                if got_ds_instance:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # dataset.\n                    rpaths = dict(zip(downloaded_paths, downloaded_paths))\n                else:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # current working directory. Take these relative to the\n                    # dataset for use with the AnnexRepo method calls.\n                    rpaths = {}\n                    for orig_path, resolved in zip(\n                            downloaded_paths,\n                            resolve_path(downloaded_paths, ds=dataset)):\n                        rpath = path_under_rev_dataset(ds, resolved)\n                        if rpath:\n                            rpaths[str(rpath)] = orig_path\n                        else:\n                            lgr.warning(\"Path %s not under dataset %s\",\n                                        orig_path, ds)\n                annex_paths = [p for p, annexed in\n                               zip(rpaths,\n                                   ds.repo.is_under_annex(list(rpaths.keys())))\n                               if annexed]\n                if annex_paths:\n                    for path in annex_paths:\n                        url = path_urls[rpaths[path]]\n                        try:\n                            # The file is already present. This is just to\n                            # register the URL.\n                            ds.repo.add_url_to_file(path, url, batch=True)\n                        except AnnexBatchCommandError as exc:\n                            lgr.warning(\"Registering %s with %s failed: %s\",\n                                        path, url, exc_str(exc))\n\n                    if archive:\n                        from datalad.api import add_archive_content\n                        for path in annex_paths:\n                            add_archive_content(path, annex=ds.repo, delete=True)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/download_url.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "e55e88dd5",
                "new_commit_hash": "058c16cc3",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/download_url.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/download_url.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 70,
                            "length": 9
                        },
                        "new_range": {
                            "start": 70,
                            "length": 13
                        },
                        "section": "class DownloadURL(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),",
                                "type": "context"
                            },
                            {
                                "content": "        path=Parameter(",
                                "type": "context"
                            },
                            {
                                "content": "            args=(\"-O\", \"--path\"),",
                                "type": "context"
                            },
                            {
                                "content": "            doc=\"path (filename or directory path) where to store downloaded file(s).  \"",
                                "type": "deleted"
                            },
                            {
                                "content": "                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"",
                                "type": "deleted"
                            },
                            {
                                "content": "                \"directory is used\",",
                                "type": "deleted"
                            },
                            {
                                "content": "            doc=\"\"\"target for download. If the path has a trailing separator,",
                                "type": "added"
                            },
                            {
                                "content": "            it is treated as a directory, and each specified URL is downloaded",
                                "type": "added"
                            },
                            {
                                "content": "            under that directory to a base name taken from the URL. Without a",
                                "type": "added"
                            },
                            {
                                "content": "            trailing separator, the value specifies the name of the downloaded",
                                "type": "added"
                            },
                            {
                                "content": "            file and only a single URL should be given. In both cases, leading",
                                "type": "added"
                            },
                            {
                                "content": "            directories will be created if needed. This argument defaults to",
                                "type": "added"
                            },
                            {
                                "content": "            the current directory.\"\"\",",
                                "type": "added"
                            },
                            {
                                "content": "            constraints=EnsureStr() | EnsureNone()),",
                                "type": "context"
                            },
                            {
                                "content": "        archive=Parameter(",
                                "type": "context"
                            },
                            {
                                "content": "            args=(\"--archive\",),",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "class",
                            "name": "DownloadURL",
                            "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            metavar='PATH',\n            doc=\"\"\"specify the dataset to add files to. If no dataset is given,\n            an attempt is made to identify the dataset based on the current\n            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to\n            prevent adding files to the dataset.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone()),\n        archive=Parameter(\n            args=(\"--archive\",),\n            action=\"store_true\",\n            doc=\"\"\"pass the downloaded files to [CMD: :command:`datalad\n            add-archive-content --delete` CMD][PY: add_archive_content(...,\n            delete=True) PY]\"\"\"),\n        save=nosave_opt,\n        message=save_message_opt\n    )\n\n    @staticmethod\n    @datasetmethod(name=\"download_url\")\n    @eval_results\n    def __call__(urls, dataset=None, path=None, overwrite=False,\n                 archive=False, save=True, message=None):\n        from ..downloaders.providers import Providers\n\n        ds = None\n        if save or dataset:\n            try:\n                ds = require_dataset(\n                    dataset, check_installed=True,\n                    purpose='downloading urls')\n            except NoDatasetArgumentFound:\n                pass\n\n        common_report = {\"action\": \"download_url\",\n                         \"ds\": ds}\n\n        got_ds_instance = isinstance(dataset, Dataset)\n        path = str(resolve_path(path or op.curdir, ds=dataset))\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not op.isdir(path):\n            yield get_status_dict(\n                status=\"error\",\n                message=(\n                    \"When specifying multiple urls, --path should point to \"\n                    \"an existing directory. Got %r\", path),\n                type=\"file\",\n                path=path,\n                **common_report)\n            return\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        path_urls = {}\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                path_urls[downloaded_path] = url\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n\n        if downloaded_paths and save and ds is not None:\n            msg = message or \"\"\"\\\n[DATALAD] Download URLs\n\nURLs:\n  {}\"\"\".format(\"\\n  \".join(urls))\n\n            for r in Save()(downloaded_paths, message=msg,\n                            # ATTN: Pass the original dataset argument to\n                            # preserve relative path handling semantics.\n                            dataset=dataset,\n                            return_type=\"generator\",\n                            result_xfm=None,\n                            result_filter=None,\n                            on_failure=\"ignore\"):\n                yield r\n\n            if isinstance(ds.repo, AnnexRepo):\n                if got_ds_instance:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # dataset.\n                    rpaths = dict(zip(downloaded_paths, downloaded_paths))\n                else:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # current working directory. Take these relative to the\n                    # dataset for use with the AnnexRepo method calls.\n                    rpaths = {}\n                    for orig_path, resolved in zip(\n                            downloaded_paths,\n                            resolve_path(downloaded_paths, ds=dataset)):\n                        rpath = path_under_rev_dataset(ds, resolved)\n                        if rpath:\n                            rpaths[str(rpath)] = orig_path\n                        else:\n                            lgr.warning(\"Path %s not under dataset %s\",\n                                        orig_path, ds)\n                annex_paths = [p for p, annexed in\n                               zip(rpaths,\n                                   ds.repo.is_under_annex(list(rpaths.keys())))\n                               if annexed]\n                if annex_paths:\n                    for path in annex_paths:\n                        url = path_urls[rpaths[path]]\n                        try:\n                            # The file is already present. This is just to\n                            # register the URL.\n                            ds.repo.add_url_to_file(path, url, batch=True)\n                        except AnnexBatchCommandError as exc:\n                            lgr.warning(\"Registering %s with %s failed: %s\",\n                                        path, url, exc_str(exc))\n\n                    if archive:\n                        from datalad.api import add_archive_content\n                        for path in annex_paths:\n                            add_archive_content(path, annex=ds.repo, delete=True)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 41,
                            "end_lineno": 202,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 104,
                            "length": 19
                        },
                        "new_range": {
                            "start": 108,
                            "length": 39
                        },
                        "section": "class DownloadURL(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                         \"ds\": ds}",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        got_ds_instance = isinstance(dataset, Dataset)",
                                "type": "context"
                            },
                            {
                                "content": "        dir_is_target = not path or path.endswith(op.sep)",
                                "type": "added"
                            },
                            {
                                "content": "        path = str(resolve_path(path or op.curdir, ds=dataset))",
                                "type": "context"
                            },
                            {
                                "content": "        if dir_is_target:",
                                "type": "added"
                            },
                            {
                                "content": "            # resolve_path() doesn't preserve trailing separators. Add one for",
                                "type": "added"
                            },
                            {
                                "content": "            # the download() call.",
                                "type": "added"
                            },
                            {
                                "content": "            path = path + op.sep",
                                "type": "added"
                            },
                            {
                                "content": "        urls = assure_list_from_str(urls)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if len(urls) > 1 and path and not op.isdir(path):",
                                "type": "deleted"
                            },
                            {
                                "content": "            yield get_status_dict(",
                                "type": "deleted"
                            },
                            {
                                "content": "                status=\"error\",",
                                "type": "deleted"
                            },
                            {
                                "content": "                message=(",
                                "type": "deleted"
                            },
                            {
                                "content": "                    \"When specifying multiple urls, --path should point to \"",
                                "type": "deleted"
                            },
                            {
                                "content": "                    \"an existing directory. Got %r\", path),",
                                "type": "deleted"
                            },
                            {
                                "content": "                type=\"file\",",
                                "type": "deleted"
                            },
                            {
                                "content": "                path=path,",
                                "type": "deleted"
                            },
                            {
                                "content": "                **common_report)",
                                "type": "deleted"
                            },
                            {
                                "content": "            return",
                                "type": "deleted"
                            },
                            {
                                "content": "        if not dir_is_target:",
                                "type": "added"
                            },
                            {
                                "content": "            if len(urls) > 1:",
                                "type": "added"
                            },
                            {
                                "content": "                yield get_status_dict(",
                                "type": "added"
                            },
                            {
                                "content": "                    status=\"error\",",
                                "type": "added"
                            },
                            {
                                "content": "                    message=(",
                                "type": "added"
                            },
                            {
                                "content": "                        \"When specifying multiple urls, --path should point to \"",
                                "type": "added"
                            },
                            {
                                "content": "                        \"a directory target (with a trailing separator). Got %r\",",
                                "type": "added"
                            },
                            {
                                "content": "                        path),",
                                "type": "added"
                            },
                            {
                                "content": "                    type=\"file\",",
                                "type": "added"
                            },
                            {
                                "content": "                    path=path,",
                                "type": "added"
                            },
                            {
                                "content": "                    **common_report)",
                                "type": "added"
                            },
                            {
                                "content": "                return",
                                "type": "added"
                            },
                            {
                                "content": "            # download() would be fine getting an existing directory and",
                                "type": "added"
                            },
                            {
                                "content": "            # downloading the URL underneath it, but let's enforce a trailing",
                                "type": "added"
                            },
                            {
                                "content": "            # slash here for consistency.",
                                "type": "added"
                            },
                            {
                                "content": "            if op.isdir(path):",
                                "type": "added"
                            },
                            {
                                "content": "                yield get_status_dict(",
                                "type": "added"
                            },
                            {
                                "content": "                    status=\"error\",",
                                "type": "added"
                            },
                            {
                                "content": "                    message=(",
                                "type": "added"
                            },
                            {
                                "content": "                        \"Non-directory path given (no trailing separator) \"",
                                "type": "added"
                            },
                            {
                                "content": "                        \"but a directory with that name exists\"),",
                                "type": "added"
                            },
                            {
                                "content": "                    type=\"file\",",
                                "type": "added"
                            },
                            {
                                "content": "                    path=path,",
                                "type": "added"
                            },
                            {
                                "content": "                    **common_report)",
                                "type": "added"
                            },
                            {
                                "content": "                return",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress",
                                "type": "context"
                            },
                            {
                                "content": "        # in % of urls which were already downloaded",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "method",
                            "name": "DownloadURL.__call__",
                            "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            metavar='PATH',\n            doc=\"\"\"specify the dataset to add files to. If no dataset is given,\n            an attempt is made to identify the dataset based on the current\n            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to\n            prevent adding files to the dataset.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone()),\n        archive=Parameter(\n            args=(\"--archive\",),\n            action=\"store_true\",\n            doc=\"\"\"pass the downloaded files to [CMD: :command:`datalad\n            add-archive-content --delete` CMD][PY: add_archive_content(...,\n            delete=True) PY]\"\"\"),\n        save=nosave_opt,\n        message=save_message_opt\n    )\n\n    @staticmethod\n    @datasetmethod(name=\"download_url\")\n    @eval_results\n    def __call__(urls, dataset=None, path=None, overwrite=False,\n                 archive=False, save=True, message=None):\n        from ..downloaders.providers import Providers\n\n        ds = None\n        if save or dataset:\n            try:\n                ds = require_dataset(\n                    dataset, check_installed=True,\n                    purpose='downloading urls')\n            except NoDatasetArgumentFound:\n                pass\n\n        common_report = {\"action\": \"download_url\",\n                         \"ds\": ds}\n\n        got_ds_instance = isinstance(dataset, Dataset)\n        path = str(resolve_path(path or op.curdir, ds=dataset))\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not op.isdir(path):\n            yield get_status_dict(\n                status=\"error\",\n                message=(\n                    \"When specifying multiple urls, --path should point to \"\n                    \"an existing directory. Got %r\", path),\n                type=\"file\",\n                path=path,\n                **common_report)\n            return\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        path_urls = {}\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                path_urls[downloaded_path] = url\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n\n        if downloaded_paths and save and ds is not None:\n            msg = message or \"\"\"\\\n[DATALAD] Download URLs\n\nURLs:\n  {}\"\"\".format(\"\\n  \".join(urls))\n\n            for r in Save()(downloaded_paths, message=msg,\n                            # ATTN: Pass the original dataset argument to\n                            # preserve relative path handling semantics.\n                            dataset=dataset,\n                            return_type=\"generator\",\n                            result_xfm=None,\n                            result_filter=None,\n                            on_failure=\"ignore\"):\n                yield r\n\n            if isinstance(ds.repo, AnnexRepo):\n                if got_ds_instance:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # dataset.\n                    rpaths = dict(zip(downloaded_paths, downloaded_paths))\n                else:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # current working directory. Take these relative to the\n                    # dataset for use with the AnnexRepo method calls.\n                    rpaths = {}\n                    for orig_path, resolved in zip(\n                            downloaded_paths,\n                            resolve_path(downloaded_paths, ds=dataset)):\n                        rpath = path_under_rev_dataset(ds, resolved)\n                        if rpath:\n                            rpaths[str(rpath)] = orig_path\n                        else:\n                            lgr.warning(\"Path %s not under dataset %s\",\n                                        orig_path, ds)\n                annex_paths = [p for p, annexed in\n                               zip(rpaths,\n                                   ds.repo.is_under_annex(list(rpaths.keys())))\n                               if annexed]\n                if annex_paths:\n                    for path in annex_paths:\n                        url = path_urls[rpaths[path]]\n                        try:\n                            # The file is already present. This is just to\n                            # register the URL.\n                            ds.repo.add_url_to_file(path, url, batch=True)\n                        except AnnexBatchCommandError as exc:\n                            lgr.warning(\"Registering %s with %s failed: %s\",\n                                        path, url, exc_str(exc))\n\n                    if archive:\n                        from datalad.api import add_archive_content\n                        for path in annex_paths:\n                            add_archive_content(path, annex=ds.repo, delete=True)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 90,
                            "end_lineno": 202,
                            "parent": {
                                "file_name": "datalad/interface/download_url.py",
                                "type": "class",
                                "name": "DownloadURL",
                                "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            metavar='PATH',\n            doc=\"\"\"specify the dataset to add files to. If no dataset is given,\n            an attempt is made to identify the dataset based on the current\n            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to\n            prevent adding files to the dataset.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone()),\n        archive=Parameter(\n            args=(\"--archive\",),\n            action=\"store_true\",\n            doc=\"\"\"pass the downloaded files to [CMD: :command:`datalad\n            add-archive-content --delete` CMD][PY: add_archive_content(...,\n            delete=True) PY]\"\"\"),\n        save=nosave_opt,\n        message=save_message_opt\n    )\n\n    @staticmethod\n    @datasetmethod(name=\"download_url\")\n    @eval_results\n    def __call__(urls, dataset=None, path=None, overwrite=False,\n                 archive=False, save=True, message=None):\n        from ..downloaders.providers import Providers\n\n        ds = None\n        if save or dataset:\n            try:\n                ds = require_dataset(\n                    dataset, check_installed=True,\n                    purpose='downloading urls')\n            except NoDatasetArgumentFound:\n                pass\n\n        common_report = {\"action\": \"download_url\",\n                         \"ds\": ds}\n\n        got_ds_instance = isinstance(dataset, Dataset)\n        path = str(resolve_path(path or op.curdir, ds=dataset))\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not op.isdir(path):\n            yield get_status_dict(\n                status=\"error\",\n                message=(\n                    \"When specifying multiple urls, --path should point to \"\n                    \"an existing directory. Got %r\", path),\n                type=\"file\",\n                path=path,\n                **common_report)\n            return\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        path_urls = {}\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                path_urls[downloaded_path] = url\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n\n        if downloaded_paths and save and ds is not None:\n            msg = message or \"\"\"\\\n[DATALAD] Download URLs\n\nURLs:\n  {}\"\"\".format(\"\\n  \".join(urls))\n\n            for r in Save()(downloaded_paths, message=msg,\n                            # ATTN: Pass the original dataset argument to\n                            # preserve relative path handling semantics.\n                            dataset=dataset,\n                            return_type=\"generator\",\n                            result_xfm=None,\n                            result_filter=None,\n                            on_failure=\"ignore\"):\n                yield r\n\n            if isinstance(ds.repo, AnnexRepo):\n                if got_ds_instance:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # dataset.\n                    rpaths = dict(zip(downloaded_paths, downloaded_paths))\n                else:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # current working directory. Take these relative to the\n                    # dataset for use with the AnnexRepo method calls.\n                    rpaths = {}\n                    for orig_path, resolved in zip(\n                            downloaded_paths,\n                            resolve_path(downloaded_paths, ds=dataset)):\n                        rpath = path_under_rev_dataset(ds, resolved)\n                        if rpath:\n                            rpaths[str(rpath)] = orig_path\n                        else:\n                            lgr.warning(\"Path %s not under dataset %s\",\n                                        orig_path, ds)\n                annex_paths = [p for p, annexed in\n                               zip(rpaths,\n                                   ds.repo.is_under_annex(list(rpaths.keys())))\n                               if annexed]\n                if annex_paths:\n                    for path in annex_paths:\n                        url = path_urls[rpaths[path]]\n                        try:\n                            # The file is already present. This is just to\n                            # register the URL.\n                            ds.repo.add_url_to_file(path, url, batch=True)\n                        except AnnexBatchCommandError as exc:\n                            lgr.warning(\"Registering %s with %s failed: %s\",\n                                        path, url, exc_str(exc))\n\n                    if archive:\n                        from datalad.api import add_archive_content\n                        for path in annex_paths:\n                            add_archive_content(path, annex=ds.repo, delete=True)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 41,
                                "end_lineno": 202,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/interface/download_url.py",
                            "type": "class",
                            "name": "DownloadURL",
                            "content": "class DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            metavar='PATH',\n            doc=\"\"\"specify the dataset to add files to. If no dataset is given,\n            an attempt is made to identify the dataset based on the current\n            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to\n            prevent adding files to the dataset.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone()),\n        archive=Parameter(\n            args=(\"--archive\",),\n            action=\"store_true\",\n            doc=\"\"\"pass the downloaded files to [CMD: :command:`datalad\n            add-archive-content --delete` CMD][PY: add_archive_content(...,\n            delete=True) PY]\"\"\"),\n        save=nosave_opt,\n        message=save_message_opt\n    )\n\n    @staticmethod\n    @datasetmethod(name=\"download_url\")\n    @eval_results\n    def __call__(urls, dataset=None, path=None, overwrite=False,\n                 archive=False, save=True, message=None):\n        from ..downloaders.providers import Providers\n\n        ds = None\n        if save or dataset:\n            try:\n                ds = require_dataset(\n                    dataset, check_installed=True,\n                    purpose='downloading urls')\n            except NoDatasetArgumentFound:\n                pass\n\n        common_report = {\"action\": \"download_url\",\n                         \"ds\": ds}\n\n        got_ds_instance = isinstance(dataset, Dataset)\n        path = str(resolve_path(path or op.curdir, ds=dataset))\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not op.isdir(path):\n            yield get_status_dict(\n                status=\"error\",\n                message=(\n                    \"When specifying multiple urls, --path should point to \"\n                    \"an existing directory. Got %r\", path),\n                type=\"file\",\n                path=path,\n                **common_report)\n            return\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        path_urls = {}\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                path_urls[downloaded_path] = url\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n\n        if downloaded_paths and save and ds is not None:\n            msg = message or \"\"\"\\\n[DATALAD] Download URLs\n\nURLs:\n  {}\"\"\".format(\"\\n  \".join(urls))\n\n            for r in Save()(downloaded_paths, message=msg,\n                            # ATTN: Pass the original dataset argument to\n                            # preserve relative path handling semantics.\n                            dataset=dataset,\n                            return_type=\"generator\",\n                            result_xfm=None,\n                            result_filter=None,\n                            on_failure=\"ignore\"):\n                yield r\n\n            if isinstance(ds.repo, AnnexRepo):\n                if got_ds_instance:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # dataset.\n                    rpaths = dict(zip(downloaded_paths, downloaded_paths))\n                else:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # current working directory. Take these relative to the\n                    # dataset for use with the AnnexRepo method calls.\n                    rpaths = {}\n                    for orig_path, resolved in zip(\n                            downloaded_paths,\n                            resolve_path(downloaded_paths, ds=dataset)):\n                        rpath = path_under_rev_dataset(ds, resolved)\n                        if rpath:\n                            rpaths[str(rpath)] = orig_path\n                        else:\n                            lgr.warning(\"Path %s not under dataset %s\",\n                                        orig_path, ds)\n                annex_paths = [p for p, annexed in\n                               zip(rpaths,\n                                   ds.repo.is_under_annex(list(rpaths.keys())))\n                               if annexed]\n                if annex_paths:\n                    for path in annex_paths:\n                        url = path_urls[rpaths[path]]\n                        try:\n                            # The file is already present. This is just to\n                            # register the URL.\n                            ds.repo.add_url_to_file(path, url, batch=True)\n                        except AnnexBatchCommandError as exc:\n                            lgr.warning(\"Registering %s with %s failed: %s\",\n                                        path, url, exc_str(exc))\n\n                    if archive:\n                        from datalad.api import add_archive_content\n                        for path in annex_paths:\n                            add_archive_content(path, annex=ds.repo, delete=True)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 41,
                            "end_lineno": 202,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for install-dataset command\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os\nfrom os.path import join as opj\n\nfrom ...api import download_url, Dataset\nfrom ...utils import chpwd\nfrom ...tests.utils import ok_, ok_exists, eq_, assert_cwd_unchanged, \\\n    assert_in, assert_false, assert_message, assert_result_count, \\\n    with_tempfile\nfrom ...tests.utils import assert_not_in\nfrom ...tests.utils import assert_in_results\nfrom ...tests.utils import with_tree\nfrom ...tests.utils import serve_path_via_http\nfrom ...tests.utils import known_failure_githubci_win\n\n\ndef test_download_url_exceptions():\n    res0 = download_url(['url1', 'url2'], path=__file__,\n                        save=False, on_failure='ignore')\n    assert_result_count(res0, 1, status='error')\n    assert_message('When specifying multiple urls, --path should point to '\n                   'an existing directory. Got %r',\n                   res0)\n\n    res1 = download_url('http://example.com/bogus',\n                        save=False, on_failure='ignore')\n    assert_result_count(res1, 1, status='error')\n    msg = res1[0]['message']\n    # when running under bogus proxy, on older systems we could get\n    # no URL reported in the message\n    if not (\n        'Cannot connect to proxy.' in msg\n        or\n        'Temporary failure in name resolution' in msg\n        or\n        'Name or service not known' in msg  # nd80\n    ):\n        assert_in('http://example.com/bogus', msg)\n\n\n@known_failure_githubci_win\n@assert_cwd_unchanged\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'abc'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_return(toppath, topurl, outdir):\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    out1 = download_url(urls[0], path=outdir, save=False)\n    assert_result_count(out1, 1)\n    eq_(out1[0]['path'], outfiles[0])\n\n    # can't overwrite\n    out2 = download_url(urls, path=outdir, on_failure='ignore', save=False)\n    assert_result_count(out2, 1, status='error')\n    assert_in('file1.txt already exists', out2[0]['message'])\n    assert_result_count(out2, 1, status='ok')  # only 2nd one\n    eq_(out2[1]['path'], outfiles[1])\n\n    out3 = download_url(urls, path=outdir, overwrite=True,\n                        on_failure='ignore', save=False)\n    assert_result_count(out3, 2, status='ok')\n    eq_([r['path'] for r in out3], outfiles)\n\n\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'def'),\n    ('file3.txt', 'ghi'),\n    ('file4.txt', 'jkl'),\n    ('file5.txt', 'mno'),\n    ('file6.txt', 'pqr'),\n    ('file7.txt', 'stu'),\n    ('file8.txt', 'vwx'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_dataset(toppath, topurl, path):\n    # Non-dataset directory.\n    file1_fullpath = opj(path, \"file1.txt\")\n    with chpwd(path):\n        download_url(topurl + \"file1.txt\")\n        ok_exists(file1_fullpath)\n    os.remove(file1_fullpath)\n\n    files_tosave = ['file1.txt', 'file2.txt']\n    urls_tosave = [topurl + f for f in files_tosave]\n\n    ds = Dataset(opj(path, \"ds\")).create()\n\n    # By default, files are saved when called in a dataset.\n    ds.download_url(urls_tosave)\n    for fname in files_tosave:\n        ok_(ds.repo.file_has_content(fname))\n\n    eq_(ds.repo.get_urls(\"file1.txt\"),\n        [urls_tosave[0]])\n    eq_(ds.repo.get_urls(\"file2.txt\"),\n        [urls_tosave[1]])\n\n    ds.download_url([topurl + \"file3.txt\"], save=False)\n    assert_false(ds.repo.file_has_content(\"file3.txt\"))\n\n    # Leading paths for target are created if needed.\n    subdir_target = opj(\"l1\", \"l2\", \"f\")\n    ds.download_url([opj(topurl, \"file1.txt\")], path=subdir_target)\n    ok_(ds.repo.file_has_content(subdir_target))\n\n    subdir_path = opj(ds.path, \"subdir\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url(topurl + \"file4.txt\")\n        download_url(topurl + \"file5.txt\", path=\"five.txt\")\n        ds.download_url(topurl + \"file6.txt\")\n        download_url(topurl + \"file7.txt\", dataset=ds.path)\n    # download_url calls within a subdirectory save the file there\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file4.txt\")))\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"five.txt\")))\n    # ... unless the dataset instance is provided\n    ok_(ds.repo.file_has_content(\"file6.txt\"))\n    # ... but a string for the dataset (as it would be from the command line)\n    # still uses CWD semantics\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file7.txt\")))\n\n    with chpwd(path):\n        # We're in a non-dataset path and pass in a string as the dataset. The\n        # path is taken as relative to the current working directory, so we get\n        # an error when trying to save it.\n        assert_in_results(\n            download_url(topurl + \"file8.txt\", dataset=ds.path,\n                         on_failure=\"ignore\"),\n            status=\"error\",\n            action=\"status\")\n    assert_false((ds.pathobj / \"file8.txt\").exists())\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive(toppath, topurl, path):\n    ds = Dataset(path).create()\n    ds.download_url([topurl + \"archive.tar.gz\"], archive=True)\n    ok_(ds.repo.file_has_content(opj(\"archive\", \"file1.txt\")))\n    assert_not_in(opj(ds.path, \"archive.tar.gz\"),\n                  ds.repo.format_commit(\"%B\"))\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive_from_subdir(toppath, topurl, path):\n    ds = Dataset(path).create()\n    subdir_path = opj(ds.path, \"subdir\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url([topurl + \"archive.tar.gz\"], archive=True)\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"archive\", \"file1.txt\")))\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for install-dataset command\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os\nfrom os.path import join as opj\n\nfrom ...api import download_url, Dataset\nfrom ...utils import chpwd\nfrom ...tests.utils import ok_, ok_exists, eq_, assert_cwd_unchanged, \\\n    assert_in, assert_false, assert_message, assert_result_count, \\\n    with_tempfile\nfrom ...tests.utils import assert_not_in\nfrom ...tests.utils import assert_in_results\nfrom ...tests.utils import with_tree\nfrom ...tests.utils import serve_path_via_http\nfrom ...tests.utils import known_failure_githubci_win\n\n\ndef test_download_url_exceptions():\n    res0 = download_url(['url1', 'url2'], path=__file__,\n                        save=False, on_failure='ignore')\n    assert_result_count(res0, 1, status='error')\n    assert_message(\"When specifying multiple urls, --path should point to \"\n                   \"a directory target (with a trailing separator). Got %r\",\n                   res0)\n\n    res1 = download_url('http://example.com/bogus',\n                        save=False, on_failure='ignore')\n    assert_result_count(res1, 1, status='error')\n    msg = res1[0]['message']\n    # when running under bogus proxy, on older systems we could get\n    # no URL reported in the message\n    if not (\n        'Cannot connect to proxy.' in msg\n        or\n        'Temporary failure in name resolution' in msg\n        or\n        'Name or service not known' in msg  # nd80\n    ):\n        assert_in('http://example.com/bogus', msg)\n\n\n@with_tree(tree={\"dir\": {}})\ndef test_download_url_existing_dir_no_slash_exception(path):\n    with chpwd(path):\n        res = download_url('url', path=\"dir\", save=False, on_failure='ignore')\n        assert_result_count(res, 1, status='error')\n        assert_message(\"Non-directory path given (no trailing separator) \"\n                       \"but a directory with that name exists\",\n                       res)\n\n\n@known_failure_githubci_win\n@assert_cwd_unchanged\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'abc'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_return(toppath, topurl, outdir):\n    # Ensure that out directory has trailing slash.\n    outdir = opj(outdir, \"\")\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    out1 = download_url(urls[0], path=outdir, save=False)\n    assert_result_count(out1, 1)\n    eq_(out1[0]['path'], outfiles[0])\n\n    # can't overwrite\n    out2 = download_url(urls, path=outdir, on_failure='ignore', save=False)\n    assert_result_count(out2, 1, status='error')\n    assert_in('file1.txt already exists', out2[0]['message'])\n    assert_result_count(out2, 1, status='ok')  # only 2nd one\n    eq_(out2[1]['path'], outfiles[1])\n\n    out3 = download_url(urls, path=outdir, overwrite=True,\n                        on_failure='ignore', save=False)\n    assert_result_count(out3, 2, status='ok')\n    eq_([r['path'] for r in out3], outfiles)\n\n\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'def'),\n    ('file3.txt', 'ghi'),\n    ('file4.txt', 'jkl'),\n    ('file5.txt', 'mno'),\n    ('file6.txt', 'pqr'),\n    ('file7.txt', 'stu'),\n    ('file8.txt', 'vwx'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_dataset(toppath, topurl, path):\n    # Non-dataset directory.\n    file1_fullpath = opj(path, \"file1.txt\")\n    with chpwd(path):\n        download_url(topurl + \"file1.txt\")\n        ok_exists(file1_fullpath)\n    os.remove(file1_fullpath)\n\n    files_tosave = ['file1.txt', 'file2.txt']\n    urls_tosave = [topurl + f for f in files_tosave]\n\n    ds = Dataset(opj(path, \"ds\")).create()\n\n    # By default, files are saved when called in a dataset.\n    ds.download_url(urls_tosave)\n    for fname in files_tosave:\n        ok_(ds.repo.file_has_content(fname))\n\n    eq_(ds.repo.get_urls(\"file1.txt\"),\n        [urls_tosave[0]])\n    eq_(ds.repo.get_urls(\"file2.txt\"),\n        [urls_tosave[1]])\n\n    ds.download_url([topurl + \"file3.txt\"], save=False)\n    assert_false(ds.repo.file_has_content(\"file3.txt\"))\n\n    # Leading paths for target are created if needed.\n    subdir_target = opj(\"l1\", \"l2\", \"f\")\n    ds.download_url([opj(topurl, \"file1.txt\")], path=subdir_target)\n    ok_(ds.repo.file_has_content(subdir_target))\n\n    subdir_path = opj(ds.path, \"subdir\", \"\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url(topurl + \"file4.txt\")\n        download_url(topurl + \"file5.txt\", path=\"five.txt\")\n        ds.download_url(topurl + \"file6.txt\")\n        download_url(topurl + \"file7.txt\", dataset=ds.path)\n    # download_url calls within a subdirectory save the file there\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file4.txt\")))\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"five.txt\")))\n    # ... unless the dataset instance is provided\n    ok_(ds.repo.file_has_content(\"file6.txt\"))\n    # ... but a string for the dataset (as it would be from the command line)\n    # still uses CWD semantics\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file7.txt\")))\n\n    with chpwd(path):\n        # We're in a non-dataset path and pass in a string as the dataset. The\n        # path is taken as relative to the current working directory, so we get\n        # an error when trying to save it.\n        assert_in_results(\n            download_url(topurl + \"file8.txt\", dataset=ds.path,\n                         on_failure=\"ignore\"),\n            status=\"error\",\n            action=\"status\")\n    assert_false((ds.pathobj / \"file8.txt\").exists())\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive(toppath, topurl, path):\n    ds = Dataset(path).create()\n    ds.download_url([topurl + \"archive.tar.gz\"], archive=True)\n    ok_(ds.repo.file_has_content(opj(\"archive\", \"file1.txt\")))\n    assert_not_in(opj(ds.path, \"archive.tar.gz\"),\n                  ds.repo.format_commit(\"%B\"))\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive_from_subdir(toppath, topurl, path):\n    ds = Dataset(path).create()\n    subdir_path = opj(ds.path, \"subdir\", \"\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url([topurl + \"archive.tar.gz\"], archive=True)\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"archive\", \"file1.txt\")))\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"a0.tar.gz\": {'f0.txt': 'abc'},\n                 \"a1.tar.gz\": {'f1.txt': 'def'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive_trailing_separator(toppath, topurl, path):\n    ds = Dataset(path).create()\n    # Archives will be extracted in the specified subdirectory, which doesn't\n    # need to exist.\n    ds.download_url([topurl + \"a0.tar.gz\"], path=opj(\"with-slash\", \"\"),\n                    archive=True)\n    ok_(ds.repo.file_has_content(opj(\"with-slash\", \"a0\", \"f0.txt\")))\n    # But if the path doesn't have a trailing separator, it will not be\n    # considered a directory. The archive will be downloaded to that path and\n    # then extracted in the top-level of the dataset.\n    ds.download_url([topurl + \"a1.tar.gz\"], path=\"no-slash\",\n                    archive=True)\n    ok_(ds.repo.file_has_content(opj(\"a1\", \"f1.txt\")))\n",
            "header": {
                "file": {
                    "path": "datalad/interface/tests/test_download_url.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "da4e50ae1",
                "new_commit_hash": "4c06814d6",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/tests/test_download_url.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/tests/test_download_url.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 31,
                            "length": 8
                        },
                        "new_range": {
                            "start": 31,
                            "length": 8
                        },
                        "section": "def test_download_url_exceptions():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    res0 = download_url(['url1', 'url2'], path=__file__,",
                                "type": "context"
                            },
                            {
                                "content": "                        save=False, on_failure='ignore')",
                                "type": "context"
                            },
                            {
                                "content": "    assert_result_count(res0, 1, status='error')",
                                "type": "context"
                            },
                            {
                                "content": "    assert_message('When specifying multiple urls, --path should point to '",
                                "type": "deleted"
                            },
                            {
                                "content": "                   'an existing directory. Got %r',",
                                "type": "deleted"
                            },
                            {
                                "content": "    assert_message(\"When specifying multiple urls, --path should point to \"",
                                "type": "added"
                            },
                            {
                                "content": "                   \"a directory target (with a trailing separator). Got %r\",",
                                "type": "added"
                            },
                            {
                                "content": "                   res0)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    res1 = download_url('http://example.com/bogus',",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_exceptions",
                            "content": "def test_download_url_exceptions():\n    res0 = download_url(['url1', 'url2'], path=__file__,\n                        save=False, on_failure='ignore')\n    assert_result_count(res0, 1, status='error')\n    assert_message('When specifying multiple urls, --path should point to '\n                   'an existing directory. Got %r',\n                   res0)\n\n    res1 = download_url('http://example.com/bogus',\n                        save=False, on_failure='ignore')\n    assert_result_count(res1, 1, status='error')\n    msg = res1[0]['message']\n    # when running under bogus proxy, on older systems we could get\n    # no URL reported in the message\n    if not (\n        'Cannot connect to proxy.' in msg\n        or\n        'Temporary failure in name resolution' in msg\n        or\n        'Name or service not known' in msg  # nd80\n    ):\n        assert_in('http://example.com/bogus', msg)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 30,
                            "end_lineno": 51,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 51,
                            "length": 6
                        },
                        "new_range": {
                            "start": 51,
                            "length": 16
                        },
                        "section": "def test_download_url_exceptions():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        assert_in('http://example.com/bogus', msg)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@with_tree(tree={\"dir\": {}})",
                                "type": "added"
                            },
                            {
                                "content": "def test_download_url_existing_dir_no_slash_exception(path):",
                                "type": "added"
                            },
                            {
                                "content": "    with chpwd(path):",
                                "type": "added"
                            },
                            {
                                "content": "        res = download_url('url', path=\"dir\", save=False, on_failure='ignore')",
                                "type": "added"
                            },
                            {
                                "content": "        assert_result_count(res, 1, status='error')",
                                "type": "added"
                            },
                            {
                                "content": "        assert_message(\"Non-directory path given (no trailing separator) \"",
                                "type": "added"
                            },
                            {
                                "content": "                       \"but a directory with that name exists\",",
                                "type": "added"
                            },
                            {
                                "content": "                       res)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "@known_failure_githubci_win",
                                "type": "context"
                            },
                            {
                                "content": "@assert_cwd_unchanged",
                                "type": "context"
                            },
                            {
                                "content": "@with_tree(tree=[",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_existing_dir_no_slash_exception",
                            "content": "def test_download_url_existing_dir_no_slash_exception(path):\n    with chpwd(path):\n        res = download_url('url', path=\"dir\", save=False, on_failure='ignore')\n        assert_result_count(res, 1, status='error')\n        assert_message(\"Non-directory path given (no trailing separator) \"\n                       \"but a directory with that name exists\",\n                       res)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 55,
                            "end_lineno": 61,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 60,
                            "length": 6
                        },
                        "new_range": {
                            "start": 70,
                            "length": 8
                        },
                        "section": "def test_download_url_exceptions():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "@serve_path_via_http",
                                "type": "context"
                            },
                            {
                                "content": "@with_tempfile(mkdir=True)",
                                "type": "context"
                            },
                            {
                                "content": "def test_download_url_return(toppath, topurl, outdir):",
                                "type": "context"
                            },
                            {
                                "content": "    # Ensure that out directory has trailing slash.",
                                "type": "added"
                            },
                            {
                                "content": "    outdir = opj(outdir, \"\")",
                                "type": "added"
                            },
                            {
                                "content": "    files = ['file1.txt', 'file2.txt']",
                                "type": "context"
                            },
                            {
                                "content": "    urls = [topurl + f for f in files]",
                                "type": "context"
                            },
                            {
                                "content": "    outfiles = [opj(outdir, f) for f in files]",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_return",
                            "content": "def test_download_url_return(toppath, topurl, outdir):\n    # Ensure that out directory has trailing slash.\n    outdir = opj(outdir, \"\")\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    out1 = download_url(urls[0], path=outdir, save=False)\n    assert_result_count(out1, 1)\n    eq_(out1[0]['path'], outfiles[0])\n\n    # can't overwrite\n    out2 = download_url(urls, path=outdir, on_failure='ignore', save=False)\n    assert_result_count(out2, 1, status='error')\n    assert_in('file1.txt already exists', out2[0]['message'])\n    assert_result_count(out2, 1, status='ok')  # only 2nd one\n    eq_(out2[1]['path'], outfiles[1])\n\n    out3 = download_url(urls, path=outdir, overwrite=True,\n                        on_failure='ignore', save=False)\n    assert_result_count(out3, 2, status='ok')\n    eq_([r['path'] for r in out3], outfiles)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 72,
                            "end_lineno": 93,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 124,
                            "length": 7
                        },
                        "new_range": {
                            "start": 136,
                            "length": 7
                        },
                        "section": "def test_download_url_dataset(toppath, topurl, path):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    ds.download_url([opj(topurl, \"file1.txt\")], path=subdir_target)",
                                "type": "context"
                            },
                            {
                                "content": "    ok_(ds.repo.file_has_content(subdir_target))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    subdir_path = opj(ds.path, \"subdir\")",
                                "type": "deleted"
                            },
                            {
                                "content": "    subdir_path = opj(ds.path, \"subdir\", \"\")",
                                "type": "added"
                            },
                            {
                                "content": "    os.mkdir(subdir_path)",
                                "type": "context"
                            },
                            {
                                "content": "    with chpwd(subdir_path):",
                                "type": "context"
                            },
                            {
                                "content": "        download_url(topurl + \"file4.txt\")",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_dataset",
                            "content": "def test_download_url_dataset(toppath, topurl, path):\n    # Non-dataset directory.\n    file1_fullpath = opj(path, \"file1.txt\")\n    with chpwd(path):\n        download_url(topurl + \"file1.txt\")\n        ok_exists(file1_fullpath)\n    os.remove(file1_fullpath)\n\n    files_tosave = ['file1.txt', 'file2.txt']\n    urls_tosave = [topurl + f for f in files_tosave]\n\n    ds = Dataset(opj(path, \"ds\")).create()\n\n    # By default, files are saved when called in a dataset.\n    ds.download_url(urls_tosave)\n    for fname in files_tosave:\n        ok_(ds.repo.file_has_content(fname))\n\n    eq_(ds.repo.get_urls(\"file1.txt\"),\n        [urls_tosave[0]])\n    eq_(ds.repo.get_urls(\"file2.txt\"),\n        [urls_tosave[1]])\n\n    ds.download_url([topurl + \"file3.txt\"], save=False)\n    assert_false(ds.repo.file_has_content(\"file3.txt\"))\n\n    # Leading paths for target are created if needed.\n    subdir_target = opj(\"l1\", \"l2\", \"f\")\n    ds.download_url([opj(topurl, \"file1.txt\")], path=subdir_target)\n    ok_(ds.repo.file_has_content(subdir_target))\n\n    subdir_path = opj(ds.path, \"subdir\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url(topurl + \"file4.txt\")\n        download_url(topurl + \"file5.txt\", path=\"five.txt\")\n        ds.download_url(topurl + \"file6.txt\")\n        download_url(topurl + \"file7.txt\", dataset=ds.path)\n    # download_url calls within a subdirectory save the file there\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file4.txt\")))\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"five.txt\")))\n    # ... unless the dataset instance is provided\n    ok_(ds.repo.file_has_content(\"file6.txt\"))\n    # ... but a string for the dataset (as it would be from the command line)\n    # still uses CWD semantics\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file7.txt\")))\n\n    with chpwd(path):\n        # We're in a non-dataset path and pass in a string as the dataset. The\n        # path is taken as relative to the current working directory, so we get\n        # an error when trying to save it.\n        assert_in_results(\n            download_url(topurl + \"file8.txt\", dataset=ds.path,\n                         on_failure=\"ignore\"),\n            status=\"error\",\n            action=\"status\")\n    assert_false((ds.pathobj / \"file8.txt\").exists())",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 96,
                            "end_lineno": 152,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 170,
                            "length": 8
                        },
                        "new_range": {
                            "start": 182,
                            "length": 28
                        },
                        "section": "def test_download_url_archive(toppath, topurl, path):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "@with_tempfile(mkdir=True)",
                                "type": "context"
                            },
                            {
                                "content": "def test_download_url_archive_from_subdir(toppath, topurl, path):",
                                "type": "context"
                            },
                            {
                                "content": "    ds = Dataset(path).create()",
                                "type": "context"
                            },
                            {
                                "content": "    subdir_path = opj(ds.path, \"subdir\")",
                                "type": "deleted"
                            },
                            {
                                "content": "    subdir_path = opj(ds.path, \"subdir\", \"\")",
                                "type": "added"
                            },
                            {
                                "content": "    os.mkdir(subdir_path)",
                                "type": "context"
                            },
                            {
                                "content": "    with chpwd(subdir_path):",
                                "type": "context"
                            },
                            {
                                "content": "        download_url([topurl + \"archive.tar.gz\"], archive=True)",
                                "type": "context"
                            },
                            {
                                "content": "    ok_(ds.repo.file_has_content(opj(\"subdir\", \"archive\", \"file1.txt\")))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "@known_failure_githubci_win",
                                "type": "added"
                            },
                            {
                                "content": "@with_tree(tree={\"a0.tar.gz\": {'f0.txt': 'abc'},",
                                "type": "added"
                            },
                            {
                                "content": "                 \"a1.tar.gz\": {'f1.txt': 'def'}})",
                                "type": "added"
                            },
                            {
                                "content": "@serve_path_via_http",
                                "type": "added"
                            },
                            {
                                "content": "@with_tempfile(mkdir=True)",
                                "type": "added"
                            },
                            {
                                "content": "def test_download_url_archive_trailing_separator(toppath, topurl, path):",
                                "type": "added"
                            },
                            {
                                "content": "    ds = Dataset(path).create()",
                                "type": "added"
                            },
                            {
                                "content": "    # Archives will be extracted in the specified subdirectory, which doesn't",
                                "type": "added"
                            },
                            {
                                "content": "    # need to exist.",
                                "type": "added"
                            },
                            {
                                "content": "    ds.download_url([topurl + \"a0.tar.gz\"], path=opj(\"with-slash\", \"\"),",
                                "type": "added"
                            },
                            {
                                "content": "                    archive=True)",
                                "type": "added"
                            },
                            {
                                "content": "    ok_(ds.repo.file_has_content(opj(\"with-slash\", \"a0\", \"f0.txt\")))",
                                "type": "added"
                            },
                            {
                                "content": "    # But if the path doesn't have a trailing separator, it will not be",
                                "type": "added"
                            },
                            {
                                "content": "    # considered a directory. The archive will be downloaded to that path and",
                                "type": "added"
                            },
                            {
                                "content": "    # then extracted in the top-level of the dataset.",
                                "type": "added"
                            },
                            {
                                "content": "    ds.download_url([topurl + \"a1.tar.gz\"], path=\"no-slash\",",
                                "type": "added"
                            },
                            {
                                "content": "                    archive=True)",
                                "type": "added"
                            },
                            {
                                "content": "    ok_(ds.repo.file_has_content(opj(\"a1\", \"f1.txt\")))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_archive_from_subdir",
                            "content": "def test_download_url_archive_from_subdir(toppath, topurl, path):\n    ds = Dataset(path).create()\n    subdir_path = opj(ds.path, \"subdir\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url([topurl + \"archive.tar.gz\"], archive=True)\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"archive\", \"file1.txt\")))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 171,
                            "end_lineno": 177,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_download_url.py",
                            "type": "function",
                            "name": "test_download_url_archive_trailing_separator",
                            "content": "def test_download_url_archive_trailing_separator(toppath, topurl, path):\n    ds = Dataset(path).create()\n    # Archives will be extracted in the specified subdirectory, which doesn't\n    # need to exist.\n    ds.download_url([topurl + \"a0.tar.gz\"], path=opj(\"with-slash\", \"\"),\n                    archive=True)\n    ok_(ds.repo.file_has_content(opj(\"with-slash\", \"a0\", \"f0.txt\")))\n    # But if the path doesn't have a trailing separator, it will not be\n    # considered a directory. The archive will be downloaded to that path and\n    # then extracted in the top-level of the dataset.\n    ds.download_url([topurl + \"a1.tar.gz\"], path=\"no-slash\",\n                    archive=True)\n    ok_(ds.repo.file_has_content(opj(\"a1\", \"f1.txt\")))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 197,
                            "end_lineno": 209,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "de9d6923181175bd33607964e0ef56deb24aa39c^",
    "new_commit_hash": "de9d6923181175bd33607964e0ef56deb24aa39c",
    "commit_message": "ENH: download_url: Use trailing separator to signal directory target\n\nIf the --path argument points to an existing directory, download_url()\nwill dump content to files within that directory.  The only way we\nknow that the user wants a directory is that one exists.  As a\nconsequence, if there's a typo (as described in gh-3484),\ndownload_url() can't be aware that a directory was intended and goes\nwith the non-directory treatment.  When combined with --archive, this\ncan lead to a large number of files in a location the user didn't\nintend, typically the top-level directory of the repository.\n\nTo improve this situation, require the user to tack on a trailing\nseparator to indicate that they want the directory treatment.  If the\nuser has a typo in the directory name, at least the content goes into\na misnamed subdirectory.  And because download_url() knows what the\nuser wanted regardless of whether the directory exists, download_url()\ncan now support creating directories when they don't exist, which\nunderneath is already supported by download().\n\nCloses #3848.",
    "commit_date": "2019-11-05T10:23:26-05:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "de9d6923181175bd33607964e0ef56deb24aa39c",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for install-dataset command\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os\nfrom os.path import join as opj\n\nfrom datalad.api import download_url, Dataset\nfrom datalad.utils import chpwd\nfrom datalad.tests.utils import ok_, ok_exists, eq_, assert_cwd_unchanged, \\\n    assert_in, assert_false, assert_message, assert_result_count, \\\n    with_tempfile\nfrom datalad.tests.utils import assert_not_in\nfrom datalad.tests.utils import assert_in_results\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import serve_path_via_http\nfrom datalad.tests.utils import known_failure_githubci_win\n\n\ndef test_download_url_exceptions():\n    res0 = download_url(['url1', 'url2'], path=__file__,\n                        save=False, on_failure='ignore')\n    assert_result_count(res0, 1, status='error')\n    assert_message(\"When specifying multiple urls, --path should point to \"\n                   \"a directory target (with a trailing separator). Got %r\",\n                   res0)\n\n    res1 = download_url('http://example.com/bogus',\n                        save=False, on_failure='ignore')\n    assert_result_count(res1, 1, status='error')\n    msg = res1[0]['message']\n    # when running under bogus proxy, on older systems we could get\n    # no URL reported in the message\n    if not (\n        'Cannot connect to proxy.' in msg\n        or\n        'Temporary failure in name resolution' in msg\n        or\n        'Name or service not known' in msg  # nd80\n    ):\n        assert_in('http://example.com/bogus', msg)\n\n\n@with_tree(tree={\"dir\": {}})\ndef test_download_url_existing_dir_no_slash_exception(path):\n    with chpwd(path):\n        res = download_url('url', path=\"dir\", save=False, on_failure='ignore')\n        assert_result_count(res, 1, status='error')\n        assert_message(\"Non-directory path given (no trailing separator) \"\n                       \"but a directory with that name exists\",\n                       res)\n\n\n@known_failure_githubci_win\n@assert_cwd_unchanged\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'abc'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_return(toppath, topurl, outdir):\n    # Ensure that out directory has trailing slash.\n    outdir = opj(outdir, \"\")\n    files = ['file1.txt', 'file2.txt']\n    urls = [topurl + f for f in files]\n    outfiles = [opj(outdir, f) for f in files]\n\n    out1 = download_url(urls[0], path=outdir, save=False)\n    assert_result_count(out1, 1)\n    eq_(out1[0]['path'], outfiles[0])\n\n    # can't overwrite\n    out2 = download_url(urls, path=outdir, on_failure='ignore', save=False)\n    assert_result_count(out2, 1, status='error')\n    assert_in('file1.txt already exists', out2[0]['message'])\n    assert_result_count(out2, 1, status='ok')  # only 2nd one\n    eq_(out2[1]['path'], outfiles[1])\n\n    out3 = download_url(urls, path=outdir, overwrite=True,\n                        on_failure='ignore', save=False)\n    assert_result_count(out3, 2, status='ok')\n    eq_([r['path'] for r in out3], outfiles)\n\n\n@with_tree(tree=[\n    ('file1.txt', 'abc'),\n    ('file2.txt', 'def'),\n    ('file3.txt', 'ghi'),\n    ('file4.txt', 'jkl'),\n    ('file5.txt', 'mno'),\n    ('file6.txt', 'pqr'),\n    ('file7.txt', 'stu'),\n    ('file8.txt', 'vwx'),\n])\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_dataset(toppath, topurl, path):\n    # Non-dataset directory.\n    file1_fullpath = opj(path, \"file1.txt\")\n    with chpwd(path):\n        download_url(topurl + \"file1.txt\")\n        ok_exists(file1_fullpath)\n    os.remove(file1_fullpath)\n\n    files_tosave = ['file1.txt', 'file2.txt']\n    urls_tosave = [topurl + f for f in files_tosave]\n\n    ds = Dataset(opj(path, \"ds\")).create()\n\n    # By default, files are saved when called in a dataset.\n    ds.download_url(urls_tosave)\n    for fname in files_tosave:\n        ok_(ds.repo.file_has_content(fname))\n\n    eq_(ds.repo.get_urls(\"file1.txt\"),\n        [urls_tosave[0]])\n    eq_(ds.repo.get_urls(\"file2.txt\"),\n        [urls_tosave[1]])\n\n    ds.download_url([topurl + \"file3.txt\"], save=False)\n    assert_false(ds.repo.file_has_content(\"file3.txt\"))\n\n    # Leading paths for target are created if needed.\n    subdir_target = opj(\"l1\", \"l2\", \"f\")\n    ds.download_url([opj(topurl, \"file1.txt\")], path=subdir_target)\n    ok_(ds.repo.file_has_content(subdir_target))\n\n    subdir_path = opj(ds.path, \"subdir\", \"\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url(topurl + \"file4.txt\")\n        download_url(topurl + \"file5.txt\", path=\"five.txt\")\n        ds.download_url(topurl + \"file6.txt\")\n        download_url(topurl + \"file7.txt\", dataset=ds.path)\n    # download_url calls within a subdirectory save the file there\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file4.txt\")))\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"five.txt\")))\n    # ... unless the dataset instance is provided\n    ok_(ds.repo.file_has_content(\"file6.txt\"))\n    # ... but a string for the dataset (as it would be from the command line)\n    # still uses CWD semantics\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"file7.txt\")))\n\n    with chpwd(path):\n        # We're in a non-dataset path and pass in a string as the dataset. The\n        # path is taken as relative to the current working directory, so we get\n        # an error when trying to save it.\n        assert_in_results(\n            download_url(topurl + \"file8.txt\", dataset=ds.path,\n                         on_failure=\"ignore\"),\n            status=\"error\",\n            action=\"status\")\n    assert_false((ds.pathobj / \"file8.txt\").exists())\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive(toppath, topurl, path):\n    ds = Dataset(path).create()\n    ds.download_url([topurl + \"archive.tar.gz\"], archive=True)\n    ok_(ds.repo.file_has_content(opj(\"archive\", \"file1.txt\")))\n    assert_not_in(opj(ds.path, \"archive.tar.gz\"),\n                  ds.repo.format_commit(\"%B\"))\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive_from_subdir(toppath, topurl, path):\n    ds = Dataset(path).create()\n    subdir_path = opj(ds.path, \"subdir\", \"\")\n    os.mkdir(subdir_path)\n    with chpwd(subdir_path):\n        download_url([topurl + \"archive.tar.gz\"], archive=True)\n    ok_(ds.repo.file_has_content(opj(\"subdir\", \"archive\", \"file1.txt\")))\n\n\n@known_failure_githubci_win\n@with_tree(tree={\"a0.tar.gz\": {'f0.txt': 'abc'},\n                 \"a1.tar.gz\": {'f1.txt': 'def'}})\n@serve_path_via_http\n@with_tempfile(mkdir=True)\ndef test_download_url_archive_trailing_separator(toppath, topurl, path):\n    ds = Dataset(path).create()\n    # Archives will be extracted in the specified subdirectory, which doesn't\n    # need to exist.\n    ds.download_url([topurl + \"a0.tar.gz\"], path=opj(\"with-slash\", \"\"),\n                    archive=True)\n    ok_(ds.repo.file_has_content(opj(\"with-slash\", \"a0\", \"f0.txt\")))\n    # But if the path doesn't have a trailing separator, it will not be\n    # considered a directory. The archive will be downloaded to that path and\n    # then extracted in the top-level of the dataset.\n    ds.download_url([topurl + \"a1.tar.gz\"], path=\"no-slash\",\n                    archive=True)\n    ok_(ds.repo.file_has_content(opj(\"a1\", \"f1.txt\")))\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.12.0rc6.dev90\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 63ms\nInstalled 25 packages in 4.34s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 66 packages in 3.18s\nPrepared 1 package in 3.75s\nInstalled 60 packages in 96ms\n + appdirs==1.4.4\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.12.0rc6 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c)\n + deprecated==1.2.15\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + gitdb==4.0.12\n + gitpython==3.1.44\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.15.0\n + pillow==11.1.0\n + propcache==0.2.1\n + pycparser==2.22\n + pygithub==2.5.0\n + pyjwt==2.10.1\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.2\n + soupsieve==2.6\n + tqdm==4.67.1\n + urllib3==1.26.20\n + vcrpy==7.0.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 100 packages in 239ms\nInstalled 32 packages in 726ms\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + filelock==3.16.1\n + gprof2dot==2024.6.6\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + nose-timer==1.0.1\n + platformdirs==4.3.6\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c\nplugins: cov-6.0.0\ncollected 7 items\n\nr2e_tests/test_1.py .FEEEEE                                              [100%]\n\n==================================== ERRORS ====================================\n__________________ ERROR at setup of test_download_url_return __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 64\n  @known_failure_githubci_win\n  @assert_cwd_unchanged\n  @with_tree(tree=[\n      ('file1.txt', 'abc'),\n      ('file2.txt', 'abc'),\n  ])\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_return(toppath, topurl, outdir):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:64\n_________________ ERROR at setup of test_download_url_dataset __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 96\n  @with_tree(tree=[\n      ('file1.txt', 'abc'),\n      ('file2.txt', 'def'),\n      ('file3.txt', 'ghi'),\n      ('file4.txt', 'jkl'),\n      ('file5.txt', 'mno'),\n      ('file6.txt', 'pqr'),\n      ('file7.txt', 'stu'),\n      ('file8.txt', 'vwx'),\n  ])\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_dataset(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:96\n_________________ ERROR at setup of test_download_url_archive __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 167\n  @known_failure_githubci_win\n  @with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_archive(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:167\n___________ ERROR at setup of test_download_url_archive_from_subdir ____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 179\n  @known_failure_githubci_win\n  @with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_archive_from_subdir(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:179\n________ ERROR at setup of test_download_url_archive_trailing_separator ________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 192\n  @known_failure_githubci_win\n  @with_tree(tree={\"a0.tar.gz\": {'f0.txt': 'abc'},\n                   \"a1.tar.gz\": {'f1.txt': 'def'}})\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_archive_trailing_separator(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:192\n=================================== FAILURES ===================================\n______________ test_download_url_existing_dir_no_slash_exception _______________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexrp69tu3v'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_download_url_existing_dir_no_slash_exception'}\nd = '/tmp/datalad_temp_tree_test_download_url_existing_dir_no_slash_exceptionsouy49pj'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_download_url_existing_dir_no_slash_exception() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:429: TypeError\n==================================== PASSES ====================================\n_________________________ test_download_url_exceptions _________________________\n----------------------------- Captured stderr call -----------------------------\n[INFO] Downloading 'http://example.com/bogus' into '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/' \n------------------------------ Captured log call -------------------------------\nINFO     datalad.downloaders:base.py:474 Downloading 'http://example.com/bogus' into '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/'\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_download_url_exceptions\nERROR r2e_tests/test_1.py::test_download_url_return\nERROR r2e_tests/test_1.py::test_download_url_dataset\nERROR r2e_tests/test_1.py::test_download_url_archive\nERROR r2e_tests/test_1.py::test_download_url_archive_from_subdir\nERROR r2e_tests/test_1.py::test_download_url_archive_trailing_separator\nFAILED r2e_tests/test_1.py::test_download_url_existing_dir_no_slash_exception\n==================== 1 failed, 1 passed, 5 errors in 2.20s =====================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c\nplugins: cov-6.0.0\ncollected 7 items\n\nr2e_tests/test_1.py FFEEEEE                                              [100%]\n\n==================================== ERRORS ====================================\n__________________ ERROR at setup of test_download_url_return __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 64\n  @known_failure_githubci_win\n  @assert_cwd_unchanged\n  @with_tree(tree=[\n      ('file1.txt', 'abc'),\n      ('file2.txt', 'abc'),\n  ])\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_return(toppath, topurl, outdir):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:64\n_________________ ERROR at setup of test_download_url_dataset __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 96\n  @with_tree(tree=[\n      ('file1.txt', 'abc'),\n      ('file2.txt', 'def'),\n      ('file3.txt', 'ghi'),\n      ('file4.txt', 'jkl'),\n      ('file5.txt', 'mno'),\n      ('file6.txt', 'pqr'),\n      ('file7.txt', 'stu'),\n      ('file8.txt', 'vwx'),\n  ])\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_dataset(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:96\n_________________ ERROR at setup of test_download_url_archive __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 167\n  @known_failure_githubci_win\n  @with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_archive(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:167\n___________ ERROR at setup of test_download_url_archive_from_subdir ____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 179\n  @known_failure_githubci_win\n  @with_tree(tree={\"archive.tar.gz\": {'file1.txt': 'abc'}})\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_archive_from_subdir(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:179\n________ ERROR at setup of test_download_url_archive_trailing_separator ________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py, line 192\n  @known_failure_githubci_win\n  @with_tree(tree={\"a0.tar.gz\": {'f0.txt': 'abc'},\n                   \"a1.tar.gz\": {'f1.txt': 'def'}})\n  @serve_path_via_http\n  @with_tempfile(mkdir=True)\n  def test_download_url_archive_trailing_separator(toppath, topurl, path):\nE       fixture 'toppath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_de9d6923181175bd33607964e0ef56deb24aa39c/r2e_tests/test_1.py:192\n=================================== FAILURES ===================================\n_________________________ test_download_url_exceptions _________________________\n\n    def test_download_url_exceptions():\n        res0 = download_url(['url1', 'url2'], path=__file__,\n                            save=False, on_failure='ignore')\n        assert_result_count(res0, 1, status='error')\n>       assert_message(\"When specifying multiple urls, --path should point to \"\n                       \"a directory target (with a trailing separator). Got %r\",\n                       res0)\n\nr2e_tests/test_1.py:34: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1268: in assert_message\n    assert_equal(m, message)\n../../../../.local/share/uv/python/cpython-3.9.21-linux-x86_64-gnu/lib/python3.9/unittest/case.py:837: in assertEqual\n    assertion_func(first, second, msg=msg)\n../../../../.local/share/uv/python/cpython-3.9.21-linux-x86_64-gnu/lib/python3.9/unittest/case.py:1217: in assertMultiLineEqual\n    self.fail(self._formatMessage(msg, standardMsg))\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <nose.tools.trivial.Dummy testMethod=nop>\nmsg = \"'When[42 chars]oint to an existing directory. Got %r' != 'When[42 chars]oint to a directory target (with a trailing s...tor). Got %r\\n?                                                                  +++++++++++++++++++++++++++++++++++\\n\"\n\n    def fail(self, msg=None):\n        \"\"\"Fail immediately, with the given message.\"\"\"\n>       raise self.failureException(msg)\nE       AssertionError: 'When[42 chars]oint to an existing directory. Got %r' != 'When[42 chars]oint to a directory target (with a trailing separator). Got %r'\nE       - When specifying multiple urls, --path should point to an existing directory. Got %r\nE       ?                                                        ----------\nE       + When specifying multiple urls, --path should point to a directory target (with a trailing separator). Got %r\nE       ?                                                                  +++++++++++++++++++++++++++++++++++\n\n../../../../.local/share/uv/python/cpython-3.9.21-linux-x86_64-gnu/lib/python3.9/unittest/case.py:676: AssertionError\n______________ test_download_url_existing_dir_no_slash_exception _______________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexbtxi1ed7'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_download_url_existing_dir_no_slash_exception'}\nd = '/tmp/datalad_temp_tree_test_download_url_existing_dir_no_slash_exceptionjce3zsbe'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_download_url_existing_dir_no_slash_exception() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:429: TypeError\n=========================== short test summary info ============================\nERROR r2e_tests/test_1.py::test_download_url_return\nERROR r2e_tests/test_1.py::test_download_url_dataset\nERROR r2e_tests/test_1.py::test_download_url_archive\nERROR r2e_tests/test_1.py::test_download_url_archive_from_subdir\nERROR r2e_tests/test_1.py::test_download_url_archive_trailing_separator\nFAILED r2e_tests/test_1.py::test_download_url_exceptions - AssertionError: 'W...\nFAILED r2e_tests/test_1.py::test_download_url_existing_dir_no_slash_exception\n========================= 2 failed, 5 errors in 0.90s ==========================\n",
    "old_commit_res_stderr": ""
}
datalad/interface/download_url.py datalad/interface/tests/test_download_url.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `download_url` Misinterprets Directory Target Without Trailing Separator

**Description:**
When using the `download_url` function with multiple URLs, specifying a directory path without a trailing separator causes unexpected behavior. Instead of recognizing the intended directory target, the function treats the path as a single file destination. This can lead to all downloaded files being placed in an unintended location, such as the repository's root directory, especially if there's a typo in the directory name.

**Example Buggy Code:**
```python
# Attempting to download multiple URLs into a directory without trailing separator
download_url(['http://example.com/file1.txt', 'http://example.com/file2.txt'], path='downloads', save=False, on_failure='ignore')
```

**Expected Behavior:**
When multiple URLs are provided, the `path` argument should clearly indicate whether it's a directory target. By adding a trailing separator (e.g., `'downloads/'`), the function should recognize it as a directory and download each file into the specified directory. If the trailing separator is omitted, the function should raise an informative error indicating that a directory target requires a trailing separator.

**Actual Behavior:**
Without the trailing separator, `download_url` does not recognize the path as a directory. Consequently, it attempts to download all specified files directly into the provided path, leading to a cluttered and unintended file structure in the repository's root or another unintended location.

**Error Message Before Fix:**
```
When specifying multiple urls, --path should point to an existing directory. Got 'downloads'
```

This error message does not clearly instruct users to add a trailing separator, causing confusion and potential misuse of the `download_url` function.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/download_url.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to DataLad downloaders
"""

__docformat__ = 'restructuredtext'

import os.path as op

from .base import Interface
from ..core.local.save import Save
from ..interface.base import build_doc
from ..interface.common_opts import nosave_opt
from ..interface.common_opts import save_message_opt
from ..interface.results import get_status_dict
from ..interface.utils import eval_results
from ..utils import assure_list_from_str
from ..distribution.dataset import Dataset
from ..distribution.dataset import datasetmethod
from ..distribution.dataset import EnsureDataset
from ..distribution.dataset import path_under_rev_dataset
from ..distribution.dataset import require_dataset
from ..distribution.dataset import resolve_path
from ..dochelpers import exc_str
from ..support.annexrepo import AnnexRepo, AnnexBatchCommandError
from ..support.param import Parameter
from ..support.constraints import EnsureStr, EnsureNone
from ..support.exceptions import NoDatasetArgumentFound

from logging import getLogger
lgr = getLogger('datalad.api.download-url')


@build_doc
class DownloadURL(Interface):
    """Download content

    It allows for a uniform download interface to various supported URL
    schemes, re-using or asking for authentication details maintained by
    datalad.

    Examples:

      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat
    """

    _params_ = dict(
        urls=Parameter(
            doc="URL(s) to be downloaded",
            constraints=EnsureStr(),  # TODO: EnsureURL
            metavar='url',
            nargs='+'),
        dataset=Parameter(
            args=("-d", "--dataset"),
            metavar='PATH',
            doc="""specify the dataset to add files to. If no dataset is given,
            an attempt is made to identify the dataset based on the current
            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to
            prevent adding files to the dataset.""",
            constraints=EnsureDataset() | EnsureNone()),
        overwrite=Parameter(
            args=("-o", "--overwrite"),
            action="store_true",
            doc="""flag to overwrite it if target file exists"""),
        path=Parameter(
            args=("-O", "--path"),
            doc="path (filename or directory path) where to store downloaded file(s).  "
                "In case of multiple URLs provided, must point to a directory.  Otherwise current "
                "directory is used",
            constraints=EnsureStr() | EnsureNone()),
        archive=Parameter(
            args=("--archive",),
            action="store_true",
            doc="""pass the downloaded files to [CMD: :command:`datalad
            add-archive-content --delete` CMD][PY: add_archive_content(...,
            delete=True) PY]"""),
        save=nosave_opt,
        message=save_message_opt
    )

    @staticmethod
    @datasetmethod(name="download_url")
    @eval_results
    def __call__(urls, dataset=None, path=None, overwrite=False,
                 archive=False, save=True, message=None):
        from ..downloaders.providers import Providers

        ds = None
        if save or dataset:
            try:
                ds = require_dataset(
                    dataset, check_installed=True,
                    purpose='downloading urls')
            except NoDatasetArgumentFound:
                pass

        common_report = {"action": "download_url",
                         "ds": ds}

        got_ds_instance = isinstance(dataset, Dataset)
        path = str(resolve_path(path or op.curdir, ds=dataset))
        urls = assure_list_from_str(urls)

        if len(urls) > 1 and path and not op.isdir(path):
            yield get_status_dict(
                status="error",
                message=(
                    "When specifying multiple urls, --path should point to "
                    "an existing directory. Got %r", path),
                type="file",
                path=path,
                **common_report)
            return

        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress
        # in % of urls which were already downloaded
        providers = Providers.from_config_files()
        downloaded_paths = []
        path_urls = {}
        for url in urls:
            # somewhat "ugly"
            # providers.get_provider(url).get_downloader(url).download(url, path=path)
            # for now -- via sugaring
            try:
                downloaded_path = providers.download(url, path=path, overwrite=overwrite)
            except Exception as e:
                yield get_status_dict(
                    status="error",
                    message=exc_str(e),
                    type="file",
                    path=path,
                    **common_report)
            else:
                downloaded_paths.append(downloaded_path)
                path_urls[downloaded_path] = url
                yield get_status_dict(
                    status="ok",
                    type="file",
                    path=downloaded_path,
                    **common_report)

        if downloaded_paths and save and ds is not None:
            msg = message or """\
[DATALAD] Download URLs

URLs:
  {}""".format("\n  ".join(urls))

            for r in Save()(downloaded_paths, message=msg,
                            # ATTN: Pass the original dataset argument to
                            # preserve relative path handling semantics.
                            dataset=dataset,
                            return_type="generator",
                            result_xfm=None,
                            result_filter=None,
                            on_failure="ignore"):
                yield r

            if isinstance(ds.repo, AnnexRepo):
                if got_ds_instance:
                    # Paths in `downloaded_paths` are already relative to the
                    # dataset.
                    rpaths = dict(zip(downloaded_paths, downloaded_paths))
                else:
                    # Paths in `downloaded_paths` are already relative to the
                    # current working directory. Take these relative to the
                    # dataset for use with the AnnexRepo method calls.
                    rpaths = {}
                    for orig_path, resolved in zip(
                            downloaded_paths,
                            resolve_path(downloaded_paths, ds=dataset)):
                        rpath = path_under_rev_dataset(ds, resolved)
                        if rpath:
                            rpaths[str(rpath)] = orig_path
                        else:
                            lgr.warning("Path %s not under dataset %s",
                                        orig_path, ds)
                annex_paths = [p for p, annexed in
                               zip(rpaths,
                                   ds.repo.is_under_annex(list(rpaths.keys())))
                               if annexed]
                if annex_paths:
                    for path in annex_paths:
                        url = path_urls[rpaths[path]]
                        try:
                            # The file is already present. This is just to
                            # register the URL.
                            ds.repo.add_url_to_file(path, url, batch=True)
                        except AnnexBatchCommandError as exc:
                            lgr.warning("Registering %s with %s failed: %s",
                                        path, url, exc_str(exc))

                    if archive:
                        from datalad.api import add_archive_content
                        for path in annex_paths:
                            add_archive_content(path, annex=ds.repo, delete=True)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** `download_url` Misinterprets Directory Target Without Trailing Separator

**Description:**
When using the `download_url` function with multiple URLs, specifying a directory path without a trailing separator causes unexpected behavior. Instead of recognizing the intended directory target, the function treats the path as a single file destination. This can lead to all downloaded files being placed in an unintended location, such as the repository's root directory, especially if there's a typo in the directory name.

**Example Buggy Code:**
```python
# Attempting to download multiple URLs into a directory without trailing separator
download_url(['http://example.com/file1.txt', 'http://example.com/file2.txt'], path='downloads', save=False, on_failure='ignore')
```

**Expected Behavior:**
When multiple URLs are provided, the `path` argument should clearly indicate whether it's a directory target. By adding a trailing separator (e.g., `'downloads/'`), the function should recognize it as a directory and download each file into the specified directory. If the trailing separator is omitted, the function should raise an informative error indicating that a directory target requires a trailing separator.

**Actual Behavior:**
Without the trailing separator, `download_url` does not recognize the path as a directory. Consequently, it attempts to download all specified files directly into the provided path, leading to a cluttered and unintended file structure in the repository's root or another unintended location.

**Error Message Before Fix:**
```
When specifying multiple urls, --path should point to an existing directory. Got 'downloads'
```

This error message does not clearly instruct users to add a trailing separator, causing confusion and potential misuse of the `download_url` function.

[/ISSUE]
{
    "test_download_url_exceptions": "PASSED",
    "test_download_url_return": "ERROR",
    "test_download_url_dataset": "ERROR",
    "test_download_url_archive": "ERROR",
    "test_download_url_archive_from_subdir": "ERROR",
    "test_download_url_archive_trailing_separator": "ERROR",
    "test_download_url_existing_dir_no_slash_exception": "FAILED"
}
diff --git a/datalad/interface/download_url.py b/datalad/interface/download_url.py
index e55e88dd5..058c16cc3 100644
--- a/datalad/interface/download_url.py
+++ b/datalad/interface/download_url.py
@@ -70,9 +70,13 @@ class DownloadURL(Interface):
             doc="""flag to overwrite it if target file exists"""),
         path=Parameter(
             args=("-O", "--path"),
-            doc="path (filename or directory path) where to store downloaded file(s).  "
-                "In case of multiple URLs provided, must point to a directory.  Otherwise current "
-                "directory is used",
+            doc="""target for download. If the path has a trailing separator,
+            it is treated as a directory, and each specified URL is downloaded
+            under that directory to a base name taken from the URL. Without a
+            trailing separator, the value specifies the name of the downloaded
+            file and only a single URL should be given. In both cases, leading
+            directories will be created if needed. This argument defaults to
+            the current directory.""",
             constraints=EnsureStr() | EnsureNone()),
         archive=Parameter(
             args=("--archive",),
@@ -104,19 +108,39 @@ class DownloadURL(Interface):
                          "ds": ds}
 
         got_ds_instance = isinstance(dataset, Dataset)
+        dir_is_target = not path or path.endswith(op.sep)
         path = str(resolve_path(path or op.curdir, ds=dataset))
+        if dir_is_target:
+            # resolve_path() doesn't preserve trailing separators. Add one for
+            # the download() call.
+            path = path + op.sep
         urls = assure_list_from_str(urls)
 
-        if len(urls) > 1 and path and not op.isdir(path):
-            yield get_status_dict(
-                status="error",
-                message=(
-                    "When specifying multiple urls, --path should point to "
-                    "an existing directory. Got %r", path),
-                type="file",
-                path=path,
-                **common_report)
-            return
+        if not dir_is_target:
+            if len(urls) > 1:
+                yield get_status_dict(
+                    status="error",
+                    message=(
+                        "When specifying multiple urls, --path should point to "
+                        "a directory target (with a trailing separator). Got %r",
+                        path),
+                    type="file",
+                    path=path,
+                    **common_report)
+                return
+            # download() would be fine getting an existing directory and
+            # downloading the URL underneath it, but let's enforce a trailing
+            # slash here for consistency.
+            if op.isdir(path):
+                yield get_status_dict(
+                    status="error",
+                    message=(
+                        "Non-directory path given (no trailing separator) "
+                        "but a directory with that name exists"),
+                    type="file",
+                    path=path,
+                    **common_report)
+                return
 
         # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress
         # in % of urls which were already downloaded

{"datalad/interface/download_url.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to DataLad downloaders\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport os.path as op\n\nfrom .base import Interface\nfrom ..core.local.save import Save\nfrom ..interface.base import build_doc\nfrom ..interface.common_opts import nosave_opt\nfrom ..interface.common_opts import save_message_opt\nfrom ..interface.results import get_status_dict\nfrom ..interface.utils import eval_results\nfrom ..utils import assure_list_from_str\nfrom ..distribution.dataset import Dataset\nfrom ..distribution.dataset import datasetmethod\nfrom ..distribution.dataset import EnsureDataset\nfrom ..distribution.dataset import path_under_rev_dataset\nfrom ..distribution.dataset import require_dataset\nfrom ..distribution.dataset import resolve_path\nfrom ..dochelpers import exc_str\nfrom ..support.annexrepo import AnnexRepo, AnnexBatchCommandError\nfrom ..support.param import Parameter\nfrom ..support.constraints import EnsureStr, EnsureNone\nfrom ..support.exceptions import NoDatasetArgumentFound\n\nfrom logging import getLogger\nlgr = getLogger('datalad.api.download-url')\n\n\n@build_doc\nclass DownloadURL(Interface):\n    \"\"\"Download content\n\n    It allows for a uniform download interface to various supported URL\n    schemes, re-using or asking for authentication details maintained by\n    datalad.\n\n    Examples:\n\n      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat\n    \"\"\"\n\n    _params_ = dict(\n        urls=Parameter(\n            doc=\"URL(s) to be downloaded\",\n            constraints=EnsureStr(),  # TODO: EnsureURL\n            metavar='url',\n            nargs='+'),\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            metavar='PATH',\n            doc=\"\"\"specify the dataset to add files to. If no dataset is given,\n            an attempt is made to identify the dataset based on the current\n            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to\n            prevent adding files to the dataset.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        overwrite=Parameter(\n            args=(\"-o\", \"--overwrite\"),\n            action=\"store_true\",\n            doc=\"\"\"flag to overwrite it if target file exists\"\"\"),\n        path=Parameter(\n            args=(\"-O\", \"--path\"),\n            doc=\"path (filename or directory path) where to store downloaded file(s).  \"\n                \"In case of multiple URLs provided, must point to a directory.  Otherwise current \"\n                \"directory is used\",\n            constraints=EnsureStr() | EnsureNone()),\n        archive=Parameter(\n            args=(\"--archive\",),\n            action=\"store_true\",\n            doc=\"\"\"pass the downloaded files to [CMD: :command:`datalad\n            add-archive-content --delete` CMD][PY: add_archive_content(...,\n            delete=True) PY]\"\"\"),\n        save=nosave_opt,\n        message=save_message_opt\n    )\n\n    @staticmethod\n    @datasetmethod(name=\"download_url\")\n    @eval_results\n    def __call__(urls, dataset=None, path=None, overwrite=False,\n                 archive=False, save=True, message=None):\n        from ..downloaders.providers import Providers\n\n        ds = None\n        if save or dataset:\n            try:\n                ds = require_dataset(\n                    dataset, check_installed=True,\n                    purpose='downloading urls')\n            except NoDatasetArgumentFound:\n                pass\n\n        common_report = {\"action\": \"download_url\",\n                         \"ds\": ds}\n\n        got_ds_instance = isinstance(dataset, Dataset)\n        path = str(resolve_path(path or op.curdir, ds=dataset))\n        urls = assure_list_from_str(urls)\n\n        if len(urls) > 1 and path and not op.isdir(path):\n            yield get_status_dict(\n                status=\"error\",\n                message=(\n                    \"When specifying multiple urls, --path should point to \"\n                    \"an existing directory. Got %r\", path),\n                type=\"file\",\n                path=path,\n                **common_report)\n            return\n\n        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress\n        # in % of urls which were already downloaded\n        providers = Providers.from_config_files()\n        downloaded_paths = []\n        path_urls = {}\n        for url in urls:\n            # somewhat \"ugly\"\n            # providers.get_provider(url).get_downloader(url).download(url, path=path)\n            # for now -- via sugaring\n            try:\n                downloaded_path = providers.download(url, path=path, overwrite=overwrite)\n            except Exception as e:\n                yield get_status_dict(\n                    status=\"error\",\n                    message=exc_str(e),\n                    type=\"file\",\n                    path=path,\n                    **common_report)\n            else:\n                downloaded_paths.append(downloaded_path)\n                path_urls[downloaded_path] = url\n                yield get_status_dict(\n                    status=\"ok\",\n                    type=\"file\",\n                    path=downloaded_path,\n                    **common_report)\n\n        if downloaded_paths and save and ds is not None:\n            msg = message or \"\"\"\\\n[DATALAD] Download URLs\n\nURLs:\n  {}\"\"\".format(\"\\n  \".join(urls))\n\n            for r in Save()(downloaded_paths, message=msg,\n                            # ATTN: Pass the original dataset argument to\n                            # preserve relative path handling semantics.\n                            dataset=dataset,\n                            return_type=\"generator\",\n                            result_xfm=None,\n                            result_filter=None,\n                            on_failure=\"ignore\"):\n                yield r\n\n            if isinstance(ds.repo, AnnexRepo):\n                if got_ds_instance:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # dataset.\n                    rpaths = dict(zip(downloaded_paths, downloaded_paths))\n                else:\n                    # Paths in `downloaded_paths` are already relative to the\n                    # current working directory. Take these relative to the\n                    # dataset for use with the AnnexRepo method calls.\n                    rpaths = {}\n                    for orig_path, resolved in zip(\n                            downloaded_paths,\n                            resolve_path(downloaded_paths, ds=dataset)):\n                        rpath = path_under_rev_dataset(ds, resolved)\n                        if rpath:\n                            rpaths[str(rpath)] = orig_path\n                        else:\n                            lgr.warning(\"Path %s not under dataset %s\",\n                                        orig_path, ds)\n                annex_paths = [p for p, annexed in\n                               zip(rpaths,\n                                   ds.repo.is_under_annex(list(rpaths.keys())))\n                               if annexed]\n                if annex_paths:\n                    for path in annex_paths:\n                        url = path_urls[rpaths[path]]\n                        try:\n                            # The file is already present. This is just to\n                            # register the URL.\n                            ds.repo.add_url_to_file(path, url, batch=True)\n                        except AnnexBatchCommandError as exc:\n                            lgr.warning(\"Registering %s with %s failed: %s\",\n                                        path, url, exc_str(exc))\n\n                    if archive:\n                        from datalad.api import add_archive_content\n                        for path in annex_paths:\n                            add_archive_content(path, annex=ds.repo, delete=True)\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `download_url` Misinterprets Directory Target Without Trailing Separator

**Description:**
When using the `download_url` function with multiple URLs, specifying a directory path without a trailing separator causes unexpected behavior. Instead of recognizing the intended directory target, the function treats the path as a single file destination. This can lead to all downloaded files being placed in an unintended location, such as the repository's root directory, especially if there's a typo in the directory name.

**Example Buggy Code:**
```python
# Attempting to download multiple URLs into a directory without trailing separator
download_url(['http://example.com/file1.txt', 'http://example.com/file2.txt'], path='downloads', save=False, on_failure='ignore')
```

**Expected Behavior:**
When multiple URLs are provided, the `path` argument should clearly indicate whether it's a directory target. By adding a trailing separator (e.g., `'downloads/'`), the function should recognize it as a directory and download each file into the specified directory. If the trailing separator is omitted, the function should raise an informative error indicating that a directory target requires a trailing separator.

**Actual Behavior:**
Without the trailing separator, `download_url` does not recognize the path as a directory. Consequently, it attempts to download all specified files directly into the provided path, leading to a cluttered and unintended file structure in the repository's root or another unintended location.

**Error Message Before Fix:**
```
When specifying multiple urls, --path should point to an existing directory. Got 'downloads'
```

This error message does not clearly instruct users to add a trailing separator, causing confusion and potential misuse of the `download_url` function.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/download_url.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to DataLad downloaders
"""

__docformat__ = 'restructuredtext'

import os.path as op

from .base import Interface
from ..core.local.save import Save
from ..interface.base import build_doc
from ..interface.common_opts import nosave_opt
from ..interface.common_opts import save_message_opt
from ..interface.results import get_status_dict
from ..interface.utils import eval_results
from ..utils import assure_list_from_str
from ..distribution.dataset import Dataset
from ..distribution.dataset import datasetmethod
from ..distribution.dataset import EnsureDataset
from ..distribution.dataset import path_under_rev_dataset
from ..distribution.dataset import require_dataset
from ..distribution.dataset import resolve_path
from ..dochelpers import exc_str
from ..support.annexrepo import AnnexRepo, AnnexBatchCommandError
from ..support.param import Parameter
from ..support.constraints import EnsureStr, EnsureNone
from ..support.exceptions import NoDatasetArgumentFound

from logging import getLogger
lgr = getLogger('datalad.api.download-url')


@build_doc
class DownloadURL(Interface):
    """Download content

    It allows for a uniform download interface to various supported URL
    schemes, re-using or asking for authentication details maintained by
    datalad.

    Examples:

      $ datalad download-url http://example.com/file.dat s3://bucket/file2.dat
    """

    _params_ = dict(
        urls=Parameter(
            doc="URL(s) to be downloaded",
            constraints=EnsureStr(),  # TODO: EnsureURL
            metavar='url',
            nargs='+'),
        dataset=Parameter(
            args=("-d", "--dataset"),
            metavar='PATH',
            doc="""specify the dataset to add files to. If no dataset is given,
            an attempt is made to identify the dataset based on the current
            working directory. Use [CMD: --nosave CMD][PY: save=False PY] to
            prevent adding files to the dataset.""",
            constraints=EnsureDataset() | EnsureNone()),
        overwrite=Parameter(
            args=("-o", "--overwrite"),
            action="store_true",
            doc="""flag to overwrite it if target file exists"""),
        path=Parameter(
            args=("-O", "--path"),
            doc="path (filename or directory path) where to store downloaded file(s).  "
                "In case of multiple URLs provided, must point to a directory.  Otherwise current "
                "directory is used",
            constraints=EnsureStr() | EnsureNone()),
        archive=Parameter(
            args=("--archive",),
            action="store_true",
            doc="""pass the downloaded files to [CMD: :command:`datalad
            add-archive-content --delete` CMD][PY: add_archive_content(...,
            delete=True) PY]"""),
        save=nosave_opt,
        message=save_message_opt
    )

    @staticmethod
    @datasetmethod(name="download_url")
    @eval_results
    def __call__(urls, dataset=None, path=None, overwrite=False,
                 archive=False, save=True, message=None):
        from ..downloaders.providers import Providers

        ds = None
        if save or dataset:
            try:
                ds = require_dataset(
                    dataset, check_installed=True,
                    purpose='downloading urls')
            except NoDatasetArgumentFound:
                pass

        common_report = {"action": "download_url",
                         "ds": ds}

        got_ds_instance = isinstance(dataset, Dataset)
        path = str(resolve_path(path or op.curdir, ds=dataset))
        urls = assure_list_from_str(urls)

        if len(urls) > 1 and path and not op.isdir(path):
            yield get_status_dict(
                status="error",
                message=(
                    "When specifying multiple urls, --path should point to "
                    "an existing directory. Got %r", path),
                type="file",
                path=path,
                **common_report)
            return

        # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress
        # in % of urls which were already downloaded
        providers = Providers.from_config_files()
        downloaded_paths = []
        path_urls = {}
        for url in urls:
            # somewhat "ugly"
            # providers.get_provider(url).get_downloader(url).download(url, path=path)
            # for now -- via sugaring
            try:
                downloaded_path = providers.download(url, path=path, overwrite=overwrite)
            except Exception as e:
                yield get_status_dict(
                    status="error",
                    message=exc_str(e),
                    type="file",
                    path=path,
                    **common_report)
            else:
                downloaded_paths.append(downloaded_path)
                path_urls[downloaded_path] = url
                yield get_status_dict(
                    status="ok",
                    type="file",
                    path=downloaded_path,
                    **common_report)

        if downloaded_paths and save and ds is not None:
            msg = message or """\
[DATALAD] Download URLs

URLs:
  {}""".format("\n  ".join(urls))

            for r in Save()(downloaded_paths, message=msg,
                            # ATTN: Pass the original dataset argument to
                            # preserve relative path handling semantics.
                            dataset=dataset,
                            return_type="generator",
                            result_xfm=None,
                            result_filter=None,
                            on_failure="ignore"):
                yield r

            if isinstance(ds.repo, AnnexRepo):
                if got_ds_instance:
                    # Paths in `downloaded_paths` are already relative to the
                    # dataset.
                    rpaths = dict(zip(downloaded_paths, downloaded_paths))
                else:
                    # Paths in `downloaded_paths` are already relative to the
                    # current working directory. Take these relative to the
                    # dataset for use with the AnnexRepo method calls.
                    rpaths = {}
                    for orig_path, resolved in zip(
                            downloaded_paths,
                            resolve_path(downloaded_paths, ds=dataset)):
                        rpath = path_under_rev_dataset(ds, resolved)
                        if rpath:
                            rpaths[str(rpath)] = orig_path
                        else:
                            lgr.warning("Path %s not under dataset %s",
                                        orig_path, ds)
                annex_paths = [p for p, annexed in
                               zip(rpaths,
                                   ds.repo.is_under_annex(list(rpaths.keys())))
                               if annexed]
                if annex_paths:
                    for path in annex_paths:
                        url = path_urls[rpaths[path]]
                        try:
                            # The file is already present. This is just to
                            # register the URL.
                            ds.repo.add_url_to_file(path, url, batch=True)
                        except AnnexBatchCommandError as exc:
                            lgr.warning("Registering %s with %s failed: %s",
                                        path, url, exc_str(exc))

                    if archive:
                        from datalad.api import add_archive_content
                        for path in annex_paths:
                            add_archive_content(path, annex=ds.repo, delete=True)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user