{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport string\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    \"\"\"\n\n    def __init__(self, idx_to_name, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n        if not isinstance(data, Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            return data[self.idx_to_name[key_int]]\n\n        try:\n            return data[key]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            field = \"tag\"\n            value = parts[0]\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value\n\n\ndef extract(stream, input_type, filename_format, url_format, meta):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta]\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, meta_args, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping, namedtuple\nfrom functools import partial\nimport logging\nimport os\nimport string\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    \"\"\"\n\n    def __init__(self, idx_to_name, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n        if not isinstance(data, Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            return data[self.idx_to_name[key_int]]\n\n        try:\n            return data[key]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            field = \"tag\"\n            value = parts[0]\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value\n\n\ndef extract(stream, input_type, filename_format, url_format, meta):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with RowInfo instance for\n    each row in `stream` and the second item is a set that contains all\n    the subdataset paths.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta]\n\n    infos = []\n    subpaths = set()\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n\n        subpath = None\n        if \"//\" in filename:\n            spaths = []\n            for part in filename.split(\"//\")[:-1]:\n                path = os.path.join(*(spaths + [part]))\n                spaths.append(path)\n                subpaths.add(path)\n            filename = filename.replace(\"//\", os.path.sep)\n            subpath = spaths[-1]\n        infos.append((filename, url, meta_args, subpath))\n    return infos, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info, subpaths = me.extract(fd, input_type,\n                                    filename_format, url_format, meta)\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        for spath in subpaths:\n            if os.path.exists(os.path.join(dataset.path, spath)):\n                lgr.warning(\n                    \"Not creating subdataset at existing path: %s\",\n                    spath)\n            else:\n                dataset.create(spath)\n\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpath in info:\n            if subpath:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                ds_current = Dataset(os.path.join(dataset.path, subpath))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "c9d8b3fe4",
                "new_commit_hash": "fc8d178f2",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 9,
                            "length": 7
                        },
                        "new_range": {
                            "start": 9,
                            "length": 7
                        },
                        "section": ""
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "\"\"\"Create and update a dataset from a list of URLs.",
                                "type": "context"
                            },
                            {
                                "content": "\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from collections import Mapping",
                                "type": "deleted"
                            },
                            {
                                "content": "from collections import Mapping, namedtuple",
                                "type": "added"
                            },
                            {
                                "content": "from functools import partial",
                                "type": "context"
                            },
                            {
                                "content": "import logging",
                                "type": "context"
                            },
                            {
                                "content": "import os",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "statement",
                            "name": "",
                            "content": "from collections import Mapping, namedtuple",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 12,
                            "end_lineno": 12,
                            "parent": null
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "statement",
                            "name": "",
                            "content": "from collections import Mapping",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 12,
                            "end_lineno": 12,
                            "parent": null
                        }
                    ]
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 102,
                            "length": 11
                        },
                        "new_range": {
                            "start": 102,
                            "length": 9
                        },
                        "section": "def extract(stream, input_type, filename_format, url_format, meta):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    Returns",
                                "type": "context"
                            },
                            {
                                "content": "    -------",
                                "type": "context"
                            },
                            {
                                "content": "    A generator that, for each item in `url_file`, yields a tuple that",
                                "type": "deleted"
                            },
                            {
                                "content": "    contains the formatted filename (with any \"//\" collapsed to a",
                                "type": "deleted"
                            },
                            {
                                "content": "    single separator), the formatted url, and a list of subdataset",
                                "type": "deleted"
                            },
                            {
                                "content": "    paths.  The length of the subdataset paths list will be equal to",
                                "type": "deleted"
                            },
                            {
                                "content": "    the number of \"//\" occurrences in the `filename_format`.",
                                "type": "deleted"
                            },
                            {
                                "content": "    A tuple where the first item is a list with RowInfo instance for",
                                "type": "added"
                            },
                            {
                                "content": "    each row in `stream` and the second item is a set that contains all",
                                "type": "added"
                            },
                            {
                                "content": "    the subdataset paths.",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    if input_type == \"csv\":",
                                "type": "context"
                            },
                            {
                                "content": "        import csv",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, filename_format, url_format, meta):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta]\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, meta_args, subpaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 93,
                            "end_lineno": 146,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 132,
                            "length": 18
                        },
                        "new_range": {
                            "start": 130,
                            "length": 25
                        },
                        "section": "def extract(stream, input_type, filename_format, url_format, meta):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    # because meta may be given multiple times on the command line.",
                                "type": "context"
                            },
                            {
                                "content": "    formats_meta = [partial(fmt.format, m) for m in meta]",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    infos = []",
                                "type": "added"
                            },
                            {
                                "content": "    subpaths = set()",
                                "type": "added"
                            },
                            {
                                "content": "    for row in rows:",
                                "type": "context"
                            },
                            {
                                "content": "        url = format_url(row)",
                                "type": "context"
                            },
                            {
                                "content": "        filename = format_filename(row)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        subpaths = []",
                                "type": "deleted"
                            },
                            {
                                "content": "        subpath = None",
                                "type": "added"
                            },
                            {
                                "content": "        if \"//\" in filename:",
                                "type": "context"
                            },
                            {
                                "content": "            spaths = []",
                                "type": "added"
                            },
                            {
                                "content": "            for part in filename.split(\"//\")[:-1]:",
                                "type": "context"
                            },
                            {
                                "content": "                subpaths.append(os.path.join(*(subpaths + [part])))",
                                "type": "deleted"
                            },
                            {
                                "content": "                path = os.path.join(*(spaths + [part]))",
                                "type": "added"
                            },
                            {
                                "content": "                spaths.append(path)",
                                "type": "added"
                            },
                            {
                                "content": "                subpaths.add(path)",
                                "type": "added"
                            },
                            {
                                "content": "            filename = filename.replace(\"//\", os.path.sep)",
                                "type": "context"
                            },
                            {
                                "content": "        yield filename, url, meta_args, subpaths",
                                "type": "deleted"
                            },
                            {
                                "content": "            subpath = spaths[-1]",
                                "type": "added"
                            },
                            {
                                "content": "        infos.append((filename, url, meta_args, subpath))",
                                "type": "added"
                            },
                            {
                                "content": "    return infos, subpaths",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, filename_format, url_format, meta):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta]\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, meta_args, subpaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 93,
                            "end_lineno": 146,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 232,
                            "length": 7
                        },
                        "new_range": {
                            "start": 237,
                            "length": 6
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    `url_file` into a \"url filename\" format that fed to 'git annex",
                                "type": "context"
                            },
                            {
                                "content": "    addurl --batch --with-files'.",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    from itertools import dropwhile",
                                "type": "deleted"
                            },
                            {
                                "content": "    import logging",
                                "type": "context"
                            },
                            {
                                "content": "    import os",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 149,
                            "end_lineno": 349,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 261,
                            "length": 9
                        },
                        "new_range": {
                            "start": 265,
                            "length": 12
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        input_type = \"json\" if extension == \".json\" else \"csv\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with open(url_file) as fd:",
                                "type": "context"
                            },
                            {
                                "content": "        info = me.extract(fd, input_type, filename_format, url_format, meta)",
                                "type": "deleted"
                            },
                            {
                                "content": "        info, subpaths = me.extract(fd, input_type,",
                                "type": "added"
                            },
                            {
                                "content": "                                    filename_format, url_format, meta)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if dry_run:",
                                "type": "context"
                            },
                            {
                                "content": "            for subpath in subpaths:",
                                "type": "added"
                            },
                            {
                                "content": "                lgr.info(\"Would create a subdataset at %s\", subpath)",
                                "type": "added"
                            },
                            {
                                "content": "            for fname, url, meta, _ in info:",
                                "type": "context"
                            },
                            {
                                "content": "                lgr.info(\"Would download %s to %s\",",
                                "type": "context"
                            },
                            {
                                "content": "                         url, os.path.join(dataset.path, fname))",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 149,
                            "end_lineno": 349,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 286,
                            "length": 28
                        },
                        "new_range": {
                            "start": 293,
                            "length": 22
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        annex_options = [\"--fast\"] if fast else []",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        seen_subpaths = set()",
                                "type": "deleted"
                            },
                            {
                                "content": "        for spath in subpaths:",
                                "type": "added"
                            },
                            {
                                "content": "            if os.path.exists(os.path.join(dataset.path, spath)):",
                                "type": "added"
                            },
                            {
                                "content": "                lgr.warning(",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Not creating subdataset at existing path: %s\",",
                                "type": "added"
                            },
                            {
                                "content": "                    spath)",
                                "type": "added"
                            },
                            {
                                "content": "            else:",
                                "type": "added"
                            },
                            {
                                "content": "                dataset.create(spath)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        files_to_add = []",
                                "type": "context"
                            },
                            {
                                "content": "        meta_to_add = []",
                                "type": "context"
                            },
                            {
                                "content": "        for fname, url, meta, subpaths in info:",
                                "type": "deleted"
                            },
                            {
                                "content": "            for spath in subpaths:",
                                "type": "deleted"
                            },
                            {
                                "content": "                if spath not in seen_subpaths:",
                                "type": "deleted"
                            },
                            {
                                "content": "                    if os.path.exists(os.path.join(dataset.path, spath)):",
                                "type": "deleted"
                            },
                            {
                                "content": "                        lgr.warning(",
                                "type": "deleted"
                            },
                            {
                                "content": "                            \"Not creating subdataset at existing path: %s\",",
                                "type": "deleted"
                            },
                            {
                                "content": "                            spath)",
                                "type": "deleted"
                            },
                            {
                                "content": "                    else:",
                                "type": "deleted"
                            },
                            {
                                "content": "                        dataset.create(spath)",
                                "type": "deleted"
                            },
                            {
                                "content": "                seen_subpaths.add(spath)",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "            if subpaths:",
                                "type": "deleted"
                            },
                            {
                                "content": "        for fname, url, meta, subpath in info:",
                                "type": "added"
                            },
                            {
                                "content": "            if subpath:",
                                "type": "added"
                            },
                            {
                                "content": "                # Adjust the dataset and filename for an `addurl` call",
                                "type": "context"
                            },
                            {
                                "content": "                # from within the subdataset that will actually contain",
                                "type": "context"
                            },
                            {
                                "content": "                # the link.",
                                "type": "context"
                            },
                            {
                                "content": "                datasets = [Dataset(os.path.join(dataset.path, sp))",
                                "type": "deleted"
                            },
                            {
                                "content": "                            for sp in subpaths]",
                                "type": "deleted"
                            },
                            {
                                "content": "                ds_current = next(dropwhile(lambda d: not d.repo,",
                                "type": "deleted"
                            },
                            {
                                "content": "                                            reversed(datasets)))",
                                "type": "deleted"
                            },
                            {
                                "content": "                ds_current = Dataset(os.path.join(dataset.path, subpath))",
                                "type": "added"
                            },
                            {
                                "content": "                ds_filename = os.path.relpath(",
                                "type": "context"
                            },
                            {
                                "content": "                    os.path.join(dataset.path, fname),",
                                "type": "context"
                            },
                            {
                                "content": "                    ds_current.path)",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 149,
                            "end_lineno": 349,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nfrom six.moves import StringIO\n\nfrom datalad.tests.utils import assert_raises\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    fmt.format(\"{0}\", values) == \"value0\"\n    fmt.format(\"{0}\", values) == fmt.format(\"{col0}\", values)\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    fmt.format(\"{notinvals}\", values, notinvals=\"ok\") == \"ok\"\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"\", \"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\" atag \"], [\"tag=atag\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        assert list(addurls.clean_meta_args(args)) == expect\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    fnames, urls, meta, subpaths = zip(*addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        [\"group={age_group}\"]))\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert meta == ([\"group=kid\"], [\"group=adult\"],\n                    [\"group=adult\"], [\"group=kid\"])\n\n    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],\n                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            [\"group={age_group}\"]]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert list(json_output) == list(csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  list,\n                  addurls.extract(None, \"not_csv_or_json\", None, None, None))\n",
            "new_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nfrom six.moves import StringIO\n\nfrom datalad.tests.utils import assert_raises\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    fmt.format(\"{0}\", values) == \"value0\"\n    fmt.format(\"{0}\", values) == fmt.format(\"{col0}\", values)\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    fmt.format(\"{notinvals}\", values, notinvals=\"ok\") == \"ok\"\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"\", \"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\" atag \"], [\"tag=atag\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        assert list(addurls.clean_meta_args(args)) == expect\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        [\"group={age_group}\"])\n\n    assert subpaths == {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"}\n\n    fnames, urls, meta, subdss = zip(*info)\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert meta == ([\"group=kid\"], [\"group=adult\"],\n                    [\"group=adult\"], [\"group=kid\"])\n\n    assert subdss == (\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\")\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            [\"group={age_group}\"]]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert json_output == csv_output\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  addurls.extract,\n                  None, \"not_csv_or_json\", None, None, None)\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/tests/test_addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "e1d7a6480",
                "new_commit_hash": "ac4875dda",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/tests/test_addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/tests/test_addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 93,
                            "length": 11
                        },
                        "new_range": {
                            "start": 93,
                            "length": 15
                        },
                        "section": "def json_stream(data):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract():",
                                "type": "context"
                            },
                            {
                                "content": "    fnames, urls, meta, subpaths = zip(*addurls.extract(",
                                "type": "deleted"
                            },
                            {
                                "content": "    info, subpaths = addurls.extract(",
                                "type": "added"
                            },
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "        [\"group={age_group}\"]))",
                                "type": "deleted"
                            },
                            {
                                "content": "        [\"group={age_group}\"])",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    assert subpaths == {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"}",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    fnames, urls, meta, subdss = zip(*info)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract",
                            "content": "def test_extract():\n    fnames, urls, meta, subpaths = zip(*addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        [\"group={age_group}\"]))\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert meta == ([\"group=kid\"], [\"group=adult\"],\n                    [\"group=adult\"], [\"group=kid\"])\n\n    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],\n                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 95,
                            "end_lineno": 111,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 107,
                            "length": 8
                        },
                        "new_range": {
                            "start": 111,
                            "length": 7
                        },
                        "section": "def test_extract():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    assert meta == ([\"group=kid\"], [\"group=adult\"],",
                                "type": "context"
                            },
                            {
                                "content": "                    [\"group=adult\"], [\"group=kid\"])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],",
                                "type": "deleted"
                            },
                            {
                                "content": "                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])",
                                "type": "deleted"
                            },
                            {
                                "content": "    assert subdss == (\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract_csv_json_equal():",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract",
                            "content": "def test_extract():\n    fnames, urls, meta, subpaths = zip(*addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        [\"group={age_group}\"]))\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert meta == ([\"group=kid\"], [\"group=adult\"],\n                    [\"group=adult\"], [\"group=kid\"])\n\n    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],\n                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 95,
                            "end_lineno": 111,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 124,
                            "length": 10
                        },
                        "new_range": {
                            "start": 127,
                            "length": 10
                        },
                        "section": "def test_extract_csv_json_equal():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)",
                                "type": "context"
                            },
                            {
                                "content": "    csv_output = addurls.extract(csv_rows, \"csv\", *args)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    assert list(json_output) == list(csv_output)",
                                "type": "deleted"
                            },
                            {
                                "content": "    assert json_output == csv_output",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract_wrong_input_type():",
                                "type": "context"
                            },
                            {
                                "content": "    assert_raises(ValueError,",
                                "type": "context"
                            },
                            {
                                "content": "                  list,",
                                "type": "deleted"
                            },
                            {
                                "content": "                  addurls.extract(None, \"not_csv_or_json\", None, None, None))",
                                "type": "deleted"
                            },
                            {
                                "content": "                  addurls.extract,",
                                "type": "added"
                            },
                            {
                                "content": "                  None, \"not_csv_or_json\", None, None, None)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_csv_json_equal",
                            "content": "def test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            [\"group={age_group}\"]]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert list(json_output) == list(csv_output)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 114,
                            "end_lineno": 127,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_wrong_input_type",
                            "content": "def test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  list,\n                  addurls.extract(None, \"not_csv_or_json\", None, None, None))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 130,
                            "end_lineno": 133,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "22500efeb3b70831836a6c7cccfc24fb9c09e8d5^",
    "new_commit_hash": "22500efeb3b70831836a6c7cccfc24fb9c09e8d5",
    "commit_message": "RF: addurls: Rework extract() to collect then return\n\nWe going to need the whole list of rows data for various\nreasons (progress bar, determination of conflicting filenames, ...),\nso just return a list up front.\n\nBecause we're doing that, we can now determine the unique set of\nsubdataset paths in extract() rather than making the caller handle\nthat.",
    "commit_date": "2018-02-27T09:07:08-05:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "22500efeb3b70831836a6c7cccfc24fb9c09e8d5",
    "test_file_codes": [
        "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nfrom six.moves import StringIO\n\nfrom datalad.tests.utils import assert_raises\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    fmt.format(\"{0}\", values) == \"value0\"\n    fmt.format(\"{0}\", values) == fmt.format(\"{col0}\", values)\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    fmt.format(\"{notinvals}\", values, notinvals=\"ok\") == \"ok\"\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"\", \"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\" atag \"], [\"tag=atag\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        assert list(addurls.clean_meta_args(args)) == expect\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        [\"group={age_group}\"])\n\n    assert subpaths == {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"}\n\n    fnames, urls, meta, subdss = zip(*info)\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert meta == ([\"group=kid\"], [\"group=adult\"],\n                    [\"group=adult\"], [\"group=kid\"])\n\n    assert subdss == (\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\")\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            [\"group={age_group}\"]]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert json_output == csv_output\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  addurls.extract,\n                  None, \"not_csv_or_json\", None, None, None)\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.9.1.dev908\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 135ms\nInstalled 25 packages in 6.28s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp5WWEfn/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp5WWEfn/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp5WWEfn/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp5WWEfn/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp5WWEfn/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpZ1Ad0l/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpZ1Ad0l/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpZ1Ad0l/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpZ1Ad0l/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpZ1Ad0l/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_22500efeb3b70831836a6c7cccfc24fb9c09e8d5/datalad/__init__.py\", line 17, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_22500efeb3b70831836a6c7cccfc24fb9c09e8d5/datalad/cmd.py\", line 35, in <module>\n    from .utils import on_windows, get_tempfile_kwargs, assure_unicode\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_22500efeb3b70831836a6c7cccfc24fb9c09e8d5/datalad/utils.py\", line 24, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 454ms\nInstalled 26 packages in 2.73s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 88 packages in 1.54s\nPrepared 1 package in 1.27s\nInstalled 77 packages in 151ms\n + appdirs==1.4.4\n + attrs==24.2.0\n + automat==22.10.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + constantly==15.1.0\n + cryptography==44.0.0\n + cssselect==1.2.0\n + datalad==0.9.1.dev1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_22500efeb3b70831836a6c7cccfc24fb9c09e8d5)\n + deprecated==1.2.15\n + duecredit==0.9.3\n + exifread==3.0.0\n + filelock==3.12.2\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + hyperlink==21.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + incremental==22.10.0\n + iso8601==2.1.0\n + itemadapter==0.8.0\n + itemloaders==1.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jmespath==1.0.1\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack-python==0.5.6\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + parsel==1.8.1\n + patool==1.12\n + pillow==9.5.0\n + protego==0.3.0\n + pyasn1==0.5.1\n + pyasn1-modules==0.3.0\n + pycparser==2.21\n + pydicom==2.4.4\n + pydispatcher==2.0.7\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyopenssl==24.3.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + queuelib==1.6.2\n + requests==2.31.0\n + requests-file==2.1.0\n + requests-ftp==0.3.1\n + scrapy==2.9.0\n + secretstorage==3.3.3\n + service-identity==21.1.0\n + simplejson==3.19.3\n + smmap==5.0.1\n + soupsieve==2.4.1\n + tldextract==4.0.0\n + tqdm==4.67.1\n + twisted==23.8.0\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + w3lib==2.1.2\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\n + zope-interface==6.4.post2\nResolved 107 packages in 224ms\nInstalled 19 packages in 67ms\n + alabaster==0.7.13\n + babel==2.14.0\n + docutils==0.19\n + imagesize==1.4.1\n + jinja2==3.1.5\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + pygments==2.17.2\n + pypandoc==1.14\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n",
    "new_commit_res_code": 0,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_22500efeb3b70831836a6c7cccfc24fb9c09e8d5\nplugins: cov-4.1.0\ncollected 9 items\n\nr2e_tests/test_1.py .........                                            [100%]\n\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_extract\nPASSED r2e_tests/test_1.py::test_extract_csv_json_equal\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\n============================== 9 passed in 0.66s ===============================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_22500efeb3b70831836a6c7cccfc24fb9c09e8d5\nplugins: cov-4.1.0\ncollected 9 items\n\nr2e_tests/test_1.py ......FFF                                            [100%]\n\n=================================== FAILURES ===================================\n_________________________________ test_extract _________________________________\n\n    def test_extract():\n        info, subpaths = addurls.extract(\n            json_stream(ST_DATA[\"rows\"]), \"json\",\n            \"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n>           [\"group={age_group}\"])\nE       ValueError: too many values to unpack (expected 2)\n\nr2e_tests/test_1.py:100: ValueError\n_________________________ test_extract_csv_json_equal __________________________\n\n    def test_extract_csv_json_equal():\n        keys = ST_DATA[\"header\"]\n        csv_rows = [\",\".join(keys)]\n        csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                        for row in ST_DATA[\"rows\"])\n    \n        args = [\"{age_group}//{now_dead}//{name}.csv\",\n                \"{name}_{debut_season}.com\",\n                [\"group={age_group}\"]]\n    \n        json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n        csv_output = addurls.extract(csv_rows, \"csv\", *args)\n    \n>       assert json_output == csv_output\nE       assert <generator ob...x79108ac0aa50> == <generator ob...x79108ac0a9d0>\nE         Use -v to get more diff\n\nr2e_tests/test_1.py:130: AssertionError\n________________________ test_extract_wrong_input_type _________________________\n\n    def test_extract_wrong_input_type():\n        assert_raises(ValueError,\n                      addurls.extract,\n>                     None, \"not_csv_or_json\", None, None, None)\n\nr2e_tests/test_1.py:136: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../../../../.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/unittest/case.py:756: in assertRaises\n    return context.handle('assertRaises', args, kwargs)\n../../../../.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/unittest/case.py:178: in handle\n    callable_obj(*args, **kwargs)\n../../../../.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/unittest/case.py:201: in __exit__\n    self.obj_name))\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <unittest.case._AssertRaisesContext object at 0x79108abc7c10>\nstandardMsg = 'ValueError not raised by extract'\n\n    def _raiseFailure(self, standardMsg):\n        msg = self.test_case._formatMessage(self.msg, standardMsg)\n>       raise self.test_case.failureException(msg)\nE       AssertionError: ValueError not raised by extract\n\n../../../../.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/unittest/case.py:135: AssertionError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nFAILED r2e_tests/test_1.py::test_extract - ValueError: too many values to unp...\nFAILED r2e_tests/test_1.py::test_extract_csv_json_equal - assert <generator o...\nFAILED r2e_tests/test_1.py::test_extract_wrong_input_type - AssertionError: V...\n========================= 3 failed, 6 passed in 0.48s ==========================\n",
    "old_commit_res_stderr": ""
}
datalad/plugin/addurls.py datalad/plugin/tests/test_addurls.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `extract()` Does Not Return Complete Data for Progress Tracking and Subdataset Management

**Description:**
The `extract()` function in `addurls.py` is currently implemented as a generator that yields individual tuples for each row. This design makes it challenging to implement features like a progress bar and to determine unique subdataset paths within the function. As a result, when attempting to unpack the return value, the application encounters issues.

**Example Buggy Code:**
```python
info, subpaths = addurls.extract(stream, input_type, filename_format, url_format, meta)
```
In this example, the code expects `extract()` to return a list of row information and a set of unique subdataset paths. However, since `extract()` is a generator that yields multiple values per row, it leads to a `ValueError` due to mismatched unpacking.

**Expected Behavior:**
The `extract()` function should return a tuple containing:
1. A list of all row data (`infos`) necessary for features like progress tracking.
2. A set of unique subdataset paths (`subpaths`) to manage conflicting filenames internally.

This would allow callers to handle the complete dataset and subdataset paths effectively without encountering unpacking errors.

**Actual Behavior:**
`extract()` returns a generator that yields tuples for each row, which does not align with the expected return structure. This causes errors when the caller tries to unpack the generator into `info` and `subpaths`, and prevents proper progress tracking and subdataset path determination within the function.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import string

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    """

    def __init__(self, idx_to_name, *args, **kwargs):
        self.idx_to_name = idx_to_name or {}
        super(Formatter, self).__init__(*args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]
        if not isinstance(data, Mapping):
            raise ValueError("First positional argument should be mapping")

        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            return data[self.idx_to_name[key_int]]

        try:
            return data[key]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)


def clean_meta_args(args):
    """Prepare formatted metadata arguments to be passed to git-annex.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    Generator that yields processed arguments (str).
    """
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            field = "tag"
            value = parts[0]

        if not value:
            # The `url_file` may have an empty value.
            continue

        yield field + "=" + value


def extract(stream, input_type, filename_format, url_format, meta):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.

    All other parameters match those described in `dlplugin`.

    Returns
    -------
    A generator that, for each item in `url_file`, yields a tuple that
    contains the formatted filename (with any "//" collapsed to a
    single separator), the formatted url, and a list of subdataset
    paths.  The length of the subdataset paths list will be equal to
    the number of "//" occurrences in the `filename_format`.
    """
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        colidx_to_name = dict(enumerate(headers))
        rows = (dict(zip(headers, r)) for r in csvrows)
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        colidx_to_name = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")

    fmt = Formatter(colidx_to_name)
    format_filename = partial(fmt.format, filename_format)
    format_url = partial(fmt.format, url_format)
    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta]

    for row in rows:
        url = format_url(row)
        filename = format_filename(row)

        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))

        subpaths = []
        if "//" in filename:
            for part in filename.split("//")[:-1]:
                subpaths.append(os.path.join(*(subpaths + [part])))
            filename = filename.replace("//", os.path.sep)
        yield filename, url, meta_args, subpaths


def dlplugin(dataset=None, url_file=None, input_type="ext",
             url_format="{0}", filename_format="{1}", meta=None,
             message=None, dry_run=False, fast=False):
    """Create and update a dataset from a list of URLs.

    Parameters
    ----------
    dataset : Dataset
        Add the URLs to this dataset (or possibly subdatasets of this
        dataset).  An empty or non-existent directory is passed to
        create a new dataset.  New subdatasets can be specified with
        `filename_format`.
    url_file : str
        A file that contains URLs or information that can be used to
        construct URLs.  Depending on the value of `input_type`, this
        should be a CSV file (with a header as the first row) or a
        JSON file (structured as a list of objects with string
        values).
    input_type : {"ext", "csv", "json"}, optional
        Whether `url_file` should be considered a CSV file or a JSON
        file.  The default value, "ext", means to consider `url_file`
        as a JSON file if it ends with ".json".  Otherwise, treat it
        as a CSV file.
    url_format : str, optional
        A format string that specifies the URL for each entry.  This
        value is similar to a normal Python format string where the
        names from `url_file` (column names for a CSV or properties
        for JSON) are available as placeholders.  If `url_file` is a
        CSV file, a positional index can also be used (i.e., "{0}" for
        the first column).  Note that a placeholder cannot contain a
        ':' or '!'.
    filename_format : str, optional
        Like `url_format`, but this format string specifies the file
        to which the URL's content will be downloaded.  The file name
        may contain directories.  The separator "//" can be used to
        indicate that the left-side directory should be created as a
        new subdataset.
    meta : str, optional
        A format string that specifies metadata.  It should be
        structured as "<field>=<value>".  The same placeholders from
        `url_format` can be used.  As an example, "location={3}" would
        mean that the value for the "location" metadata field should
        be set the value of the fourth column.  A plain value is
        shorthand for "tag=<value>".  This option can be given
        multiple times.
    message : str, optional
        Use this message when committing the URL additions.
    dry_run : bool, optional
        Report which URLs would be downloaded to which files and then
        exit.
    fast : bool, optional
        If True, add the URLs, but don't download their content.
        Underneath, this passes the --fast flag to `git annex addurl`.

    Examples
    --------
    Consider a file "avatars.csv" that contains

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and
    'ext' fields, we could run

        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv
          url_format='{link}' filename_format='{who}.{ext}' fast=True

    The '-d avatar_ds' is used to create a new dataset in
    "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new
    subdataset in an "avatars" subdirectory, we could use "//" in the
    `filename_format` argument:

        $ datalad plugin addurls url_file=avatars.csv
          url_format='{link}' filename_format='avatars//{who}.{ext}'
          fast=True

    Note
    ----
    For users familiar with 'git annex addurl': A large part of this
    plugin's functionality can be viewed as transforming data from
    `url_file` into a "url filename" format that fed to 'git annex
    addurl --batch --with-files'.
    """
    from itertools import dropwhile
    import logging
    import os

    from datalad.distribution.dataset import Dataset
    from datalad.interface.results import get_status_dict
    import datalad.plugin.addurls as me
    from datalad.support.annexrepo import AnnexRepo
    from datalad.utils import assure_list

    lgr = logging.getLogger("datalad.plugin.addurls")

    meta = assure_list(meta)

    if url_file is None:
        # `url_file` is not a required argument in `dlplugin` because
        # the argument before it, `dataset`, needs to be optional to
        # support the creation of new datasets.
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="Must specify url_file argument")
        return

    if input_type == "ext":
        extension = os.path.splitext(url_file)[1]
        input_type = "json" if extension == ".json" else "csv"

    with open(url_file) as fd:
        info = me.extract(fd, input_type, filename_format, url_format, meta)

        if dry_run:
            for fname, url, meta, _ in info:
                lgr.info("Would download %s to %s",
                         url, os.path.join(dataset.path, fname))
                lgr.info("Metadata: %s", meta)
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="ok",
                                  message="dry-run finished")
            return

        if not dataset.repo:
            # Populate a new dataset with the URLs.
            dataset.create()
        elif not isinstance(dataset.repo, AnnexRepo):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message="not an annex repo")
            return

        annex_options = ["--fast"] if fast else []

        seen_subpaths = set()
        files_to_add = []
        meta_to_add = []
        for fname, url, meta, subpaths in info:
            for spath in subpaths:
                if spath not in seen_subpaths:
                    if os.path.exists(os.path.join(dataset.path, spath)):
                        lgr.warning(
                            "Not creating subdataset at existing path: %s",
                            spath)
                    else:
                        dataset.create(spath)
                seen_subpaths.add(spath)

            if subpaths:
                # Adjust the dataset and filename for an `addurl` call
                # from within the subdataset that will actually contain
                # the link.
                datasets = [Dataset(os.path.join(dataset.path, sp))
                            for sp in subpaths]
                ds_current = next(dropwhile(lambda d: not d.repo,
                                            reversed(datasets)))
                ds_filename = os.path.relpath(
                    os.path.join(dataset.path, fname),
                    ds_current.path)
            else:
                ds_current = dataset
                ds_filename = fname

            ds_current.repo.add_url_to_file(ds_filename, url,
                                            batch=True, options=annex_options)
            yield get_status_dict(action="addurls",
                                  ds=ds_current,
                                  type="file",
                                  path=os.path.join(ds_current.path,
                                                    ds_filename),
                                  status="ok")

            files_to_add.append(fname)
            meta_to_add.append((ds_current, ds_filename, meta))

        msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)
        for r in dataset.add(files_to_add, message=msg):
            yield r

        for ds, fname, meta in meta_to_add:
            lgr.debug("Adding metadata to %s in %s", fname, ds.path)
            for arg in meta:
                ds.repo._run_annex_command("metadata",
                                           annex_options=["--set", arg, fname])
            yield get_status_dict(action="addurls-metadata",
                                  ds=ds_current,
                                  type="file",
                                  path=os.path.join(ds.path, fname),
                                  message="added metadata",
                                  status="ok")

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** `extract()` Does Not Return Complete Data for Progress Tracking and Subdataset Management

**Description:**
The `extract()` function in `addurls.py` is currently implemented as a generator that yields individual tuples for each row. This design makes it challenging to implement features like a progress bar and to determine unique subdataset paths within the function. As a result, when attempting to unpack the return value, the application encounters issues.

**Example Buggy Code:**
```python
info, subpaths = addurls.extract(stream, input_type, filename_format, url_format, meta)
```
In this example, the code expects `extract()` to return a list of row information and a set of unique subdataset paths. However, since `extract()` is a generator that yields multiple values per row, it leads to a `ValueError` due to mismatched unpacking.

**Expected Behavior:**
The `extract()` function should return a tuple containing:
1. A list of all row data (`infos`) necessary for features like progress tracking.
2. A set of unique subdataset paths (`subpaths`) to manage conflicting filenames internally.

This would allow callers to handle the complete dataset and subdataset paths effectively without encountering unpacking errors.

**Actual Behavior:**
`extract()` returns a generator that yields tuples for each row, which does not align with the expected return structure. This causes errors when the caller tries to unpack the generator into `info` and `subpaths`, and prevents proper progress tracking and subdataset path determination within the function.

[/ISSUE]
{
    "test_formatter": "PASSED",
    "test_formatter_no_idx_map": "PASSED",
    "test_formatter_no_mapping_arg": "PASSED",
    "test_formatter_placeholder_with_spaces": "PASSED",
    "test_formatter_placeholder_nonpermitted_chars": "PASSED",
    "test_clean_meta_args": "PASSED",
    "test_extract": "PASSED",
    "test_extract_csv_json_equal": "PASSED",
    "test_extract_wrong_input_type": "PASSED"
}
diff --git a/datalad/plugin/addurls.py b/datalad/plugin/addurls.py
index c9d8b3fe4..fc8d178f2 100644
--- a/datalad/plugin/addurls.py
+++ b/datalad/plugin/addurls.py
@@ -9,7 +9,7 @@
 """Create and update a dataset from a list of URLs.
 """
 
-from collections import Mapping
+from collections import Mapping, namedtuple
 from functools import partial
 import logging
 import os
@@ -102,11 +102,9 @@ def extract(stream, input_type, filename_format, url_format, meta):
 
     Returns
     -------
-    A generator that, for each item in `url_file`, yields a tuple that
-    contains the formatted filename (with any "//" collapsed to a
-    single separator), the formatted url, and a list of subdataset
-    paths.  The length of the subdataset paths list will be equal to
-    the number of "//" occurrences in the `filename_format`.
+    A tuple where the first item is a list with RowInfo instance for
+    each row in `stream` and the second item is a set that contains all
+    the subdataset paths.
     """
     if input_type == "csv":
         import csv
@@ -132,18 +130,25 @@ def extract(stream, input_type, filename_format, url_format, meta):
     # because meta may be given multiple times on the command line.
     formats_meta = [partial(fmt.format, m) for m in meta]
 
+    infos = []
+    subpaths = set()
     for row in rows:
         url = format_url(row)
         filename = format_filename(row)
 
         meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))
 
-        subpaths = []
+        subpath = None
         if "//" in filename:
+            spaths = []
             for part in filename.split("//")[:-1]:
-                subpaths.append(os.path.join(*(subpaths + [part])))
+                path = os.path.join(*(spaths + [part]))
+                spaths.append(path)
+                subpaths.add(path)
             filename = filename.replace("//", os.path.sep)
-        yield filename, url, meta_args, subpaths
+            subpath = spaths[-1]
+        infos.append((filename, url, meta_args, subpath))
+    return infos, subpaths
 
 
 def dlplugin(dataset=None, url_file=None, input_type="ext",
@@ -232,7 +237,6 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
     `url_file` into a "url filename" format that fed to 'git annex
     addurl --batch --with-files'.
     """
-    from itertools import dropwhile
     import logging
     import os
 
@@ -261,9 +265,12 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
         input_type = "json" if extension == ".json" else "csv"
 
     with open(url_file) as fd:
-        info = me.extract(fd, input_type, filename_format, url_format, meta)
+        info, subpaths = me.extract(fd, input_type,
+                                    filename_format, url_format, meta)
 
         if dry_run:
+            for subpath in subpaths:
+                lgr.info("Would create a subdataset at %s", subpath)
             for fname, url, meta, _ in info:
                 lgr.info("Would download %s to %s",
                          url, os.path.join(dataset.path, fname))
@@ -286,28 +293,22 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
 
         annex_options = ["--fast"] if fast else []
 
-        seen_subpaths = set()
+        for spath in subpaths:
+            if os.path.exists(os.path.join(dataset.path, spath)):
+                lgr.warning(
+                    "Not creating subdataset at existing path: %s",
+                    spath)
+            else:
+                dataset.create(spath)
+
         files_to_add = []
         meta_to_add = []
-        for fname, url, meta, subpaths in info:
-            for spath in subpaths:
-                if spath not in seen_subpaths:
-                    if os.path.exists(os.path.join(dataset.path, spath)):
-                        lgr.warning(
-                            "Not creating subdataset at existing path: %s",
-                            spath)
-                    else:
-                        dataset.create(spath)
-                seen_subpaths.add(spath)
-
-            if subpaths:
+        for fname, url, meta, subpath in info:
+            if subpath:
                 # Adjust the dataset and filename for an `addurl` call
                 # from within the subdataset that will actually contain
                 # the link.
-                datasets = [Dataset(os.path.join(dataset.path, sp))
-                            for sp in subpaths]
-                ds_current = next(dropwhile(lambda d: not d.repo,
-                                            reversed(datasets)))
+                ds_current = Dataset(os.path.join(dataset.path, subpath))
                 ds_filename = os.path.relpath(
                     os.path.join(dataset.path, fname),
                     ds_current.path)

{"datalad/plugin/addurls.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport string\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    \"\"\"\n\n    def __init__(self, idx_to_name, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n        if not isinstance(data, Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            return data[self.idx_to_name[key_int]]\n\n        try:\n            return data[key]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            field = \"tag\"\n            value = parts[0]\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value\n\n\ndef extract(stream, input_type, filename_format, url_format, meta):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta]\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, meta_args, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `extract()` Does Not Return Complete Data for Progress Tracking and Subdataset Management

**Description:**
The `extract()` function in `addurls.py` is currently implemented as a generator that yields individual tuples for each row. This design makes it challenging to implement features like a progress bar and to determine unique subdataset paths within the function. As a result, when attempting to unpack the return value, the application encounters issues.

**Example Buggy Code:**
```python
info, subpaths = addurls.extract(stream, input_type, filename_format, url_format, meta)
```
In this example, the code expects `extract()` to return a list of row information and a set of unique subdataset paths. However, since `extract()` is a generator that yields multiple values per row, it leads to a `ValueError` due to mismatched unpacking.

**Expected Behavior:**
The `extract()` function should return a tuple containing:
1. A list of all row data (`infos`) necessary for features like progress tracking.
2. A set of unique subdataset paths (`subpaths`) to manage conflicting filenames internally.

This would allow callers to handle the complete dataset and subdataset paths effectively without encountering unpacking errors.

**Actual Behavior:**
`extract()` returns a generator that yields tuples for each row, which does not align with the expected return structure. This causes errors when the caller tries to unpack the generator into `info` and `subpaths`, and prevents proper progress tracking and subdataset path determination within the function.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import string

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    """

    def __init__(self, idx_to_name, *args, **kwargs):
        self.idx_to_name = idx_to_name or {}
        super(Formatter, self).__init__(*args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]
        if not isinstance(data, Mapping):
            raise ValueError("First positional argument should be mapping")

        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            return data[self.idx_to_name[key_int]]

        try:
            return data[key]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)


def clean_meta_args(args):
    """Prepare formatted metadata arguments to be passed to git-annex.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    Generator that yields processed arguments (str).
    """
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            field = "tag"
            value = parts[0]

        if not value:
            # The `url_file` may have an empty value.
            continue

        yield field + "=" + value


def extract(stream, input_type, filename_format, url_format, meta):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.

    All other parameters match those described in `dlplugin`.

    Returns
    -------
    A generator that, for each item in `url_file`, yields a tuple that
    contains the formatted filename (with any "//" collapsed to a
    single separator), the formatted url, and a list of subdataset
    paths.  The length of the subdataset paths list will be equal to
    the number of "//" occurrences in the `filename_format`.
    """
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        colidx_to_name = dict(enumerate(headers))
        rows = (dict(zip(headers, r)) for r in csvrows)
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        colidx_to_name = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")

    fmt = Formatter(colidx_to_name)
    format_filename = partial(fmt.format, filename_format)
    format_url = partial(fmt.format, url_format)
    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta]

    for row in rows:
        url = format_url(row)
        filename = format_filename(row)

        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))

        subpaths = []
        if "//" in filename:
            for part in filename.split("//")[:-1]:
                subpaths.append(os.path.join(*(subpaths + [part])))
            filename = filename.replace("//", os.path.sep)
        yield filename, url, meta_args, subpaths


def dlplugin(dataset=None, url_file=None, input_type="ext",
             url_format="{0}", filename_format="{1}", meta=None,
             message=None, dry_run=False, fast=False):
    """Create and update a dataset from a list of URLs.

    Parameters
    ----------
    dataset : Dataset
        Add the URLs to this dataset (or possibly subdatasets of this
        dataset).  An empty or non-existent directory is passed to
        create a new dataset.  New subdatasets can be specified with
        `filename_format`.
    url_file : str
        A file that contains URLs or information that can be used to
        construct URLs.  Depending on the value of `input_type`, this
        should be a CSV file (with a header as the first row) or a
        JSON file (structured as a list of objects with string
        values).
    input_type : {"ext", "csv", "json"}, optional
        Whether `url_file` should be considered a CSV file or a JSON
        file.  The default value, "ext", means to consider `url_file`
        as a JSON file if it ends with ".json".  Otherwise, treat it
        as a CSV file.
    url_format : str, optional
        A format string that specifies the URL for each entry.  This
        value is similar to a normal Python format string where the
        names from `url_file` (column names for a CSV or properties
        for JSON) are available as placeholders.  If `url_file` is a
        CSV file, a positional index can also be used (i.e., "{0}" for
        the first column).  Note that a placeholder cannot contain a
        ':' or '!'.
    filename_format : str, optional
        Like `url_format`, but this format string specifies the file
        to which the URL's content will be downloaded.  The file name
        may contain directories.  The separator "//" can be used to
        indicate that the left-side directory should be created as a
        new subdataset.
    meta : str, optional
        A format string that specifies metadata.  It should be
        structured as "<field>=<value>".  The same placeholders from
        `url_format` can be used.  As an example, "location={3}" would
        mean that the value for the "location" metadata field should
        be set the value of the fourth column.  A plain value is
        shorthand for "tag=<value>".  This option can be given
        multiple times.
    message : str, optional
        Use this message when committing the URL additions.
    dry_run : bool, optional
        Report which URLs would be downloaded to which files and then
        exit.
    fast : bool, optional
        If True, add the URLs, but don't download their content.
        Underneath, this passes the --fast flag to `git annex addurl`.

    Examples
    --------
    Consider a file "avatars.csv" that contains

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and
    'ext' fields, we could run

        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv
          url_format='{link}' filename_format='{who}.{ext}' fast=True

    The '-d avatar_ds' is used to create a new dataset in
    "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new
    subdataset in an "avatars" subdirectory, we could use "//" in the
    `filename_format` argument:

        $ datalad plugin addurls url_file=avatars.csv
          url_format='{link}' filename_format='avatars//{who}.{ext}'
          fast=True

    Note
    ----
    For users familiar with 'git annex addurl': A large part of this
    plugin's functionality can be viewed as transforming data from
    `url_file` into a "url filename" format that fed to 'git annex
    addurl --batch --with-files'.
    """
    from itertools import dropwhile
    import logging
    import os

    from datalad.distribution.dataset import Dataset
    from datalad.interface.results import get_status_dict
    import datalad.plugin.addurls as me
    from datalad.support.annexrepo import AnnexRepo
    from datalad.utils import assure_list

    lgr = logging.getLogger("datalad.plugin.addurls")

    meta = assure_list(meta)

    if url_file is None:
        # `url_file` is not a required argument in `dlplugin` because
        # the argument before it, `dataset`, needs to be optional to
        # support the creation of new datasets.
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="Must specify url_file argument")
        return

    if input_type == "ext":
        extension = os.path.splitext(url_file)[1]
        input_type = "json" if extension == ".json" else "csv"

    with open(url_file) as fd:
        info = me.extract(fd, input_type, filename_format, url_format, meta)

        if dry_run:
            for fname, url, meta, _ in info:
                lgr.info("Would download %s to %s",
                         url, os.path.join(dataset.path, fname))
                lgr.info("Metadata: %s", meta)
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="ok",
                                  message="dry-run finished")
            return

        if not dataset.repo:
            # Populate a new dataset with the URLs.
            dataset.create()
        elif not isinstance(dataset.repo, AnnexRepo):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message="not an annex repo")
            return

        annex_options = ["--fast"] if fast else []

        seen_subpaths = set()
        files_to_add = []
        meta_to_add = []
        for fname, url, meta, subpaths in info:
            for spath in subpaths:
                if spath not in seen_subpaths:
                    if os.path.exists(os.path.join(dataset.path, spath)):
                        lgr.warning(
                            "Not creating subdataset at existing path: %s",
                            spath)
                    else:
                        dataset.create(spath)
                seen_subpaths.add(spath)

            if subpaths:
                # Adjust the dataset and filename for an `addurl` call
                # from within the subdataset that will actually contain
                # the link.
                datasets = [Dataset(os.path.join(dataset.path, sp))
                            for sp in subpaths]
                ds_current = next(dropwhile(lambda d: not d.repo,
                                            reversed(datasets)))
                ds_filename = os.path.relpath(
                    os.path.join(dataset.path, fname),
                    ds_current.path)
            else:
                ds_current = dataset
                ds_filename = fname

            ds_current.repo.add_url_to_file(ds_filename, url,
                                            batch=True, options=annex_options)
            yield get_status_dict(action="addurls",
                                  ds=ds_current,
                                  type="file",
                                  path=os.path.join(ds_current.path,
                                                    ds_filename),
                                  status="ok")

            files_to_add.append(fname)
            meta_to_add.append((ds_current, ds_filename, meta))

        msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)
        for r in dataset.add(files_to_add, message=msg):
            yield r

        for ds, fname, meta in meta_to_add:
            lgr.debug("Adding metadata to %s in %s", fname, ds.path)
            for arg in meta:
                ds.repo._run_annex_command("metadata",
                                           annex_options=["--set", arg, fname])
            yield get_status_dict(action="addurls-metadata",
                                  ds=ds_current,
                                  type="file",
                                  path=os.path.join(ds.path, fname),
                                  message="added metadata",
                                  status="ok")

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user