{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport string\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    \"\"\"\n\n    def __init__(self, idx_to_name, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n        if not isinstance(data, Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            return data[self.idx_to_name[key_int]]\n\n        try:\n            return data[key]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n\ndef extract(stream, input_type, filename_format, url_format):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format)\n\n        if dry_run:\n            for fname, url, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        to_add = []\n        for fname, url, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            to_add.append(fname)\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(to_add, message=msg):\n            yield r\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport string\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    \"\"\"\n\n    def __init__(self, idx_to_name, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n        if not isinstance(data, Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            return data[self.idx_to_name[key_int]]\n\n        try:\n            return data[key]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            field = \"tag\"\n            value = parts[0]\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value\n\n\ndef extract(stream, input_type, filename_format, url_format, meta):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta]\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, meta_args, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "a10d6f1e7",
                "new_commit_hash": "c9d8b3fe4",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 61,
                            "length": 7
                        },
                        "new_range": {
                            "start": 61,
                            "length": 36
                        },
                        "section": "class Formatter(string.Formatter):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                key, args, kwargs)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def extract(stream, input_type, filename_format, url_format):",
                                "type": "deleted"
                            },
                            {
                                "content": "def clean_meta_args(args):",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Parameters",
                                "type": "added"
                            },
                            {
                                "content": "    ----------",
                                "type": "added"
                            },
                            {
                                "content": "    args : iterable of str",
                                "type": "added"
                            },
                            {
                                "content": "        Formatted metadata arguments for 'git-annex metadata --set'.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Returns",
                                "type": "added"
                            },
                            {
                                "content": "    -------",
                                "type": "added"
                            },
                            {
                                "content": "    Generator that yields processed arguments (str).",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "    for arg in args:",
                                "type": "added"
                            },
                            {
                                "content": "        parts = [x.strip() for x in arg.split(\"=\", 1)]",
                                "type": "added"
                            },
                            {
                                "content": "        if len(parts) == 2:",
                                "type": "added"
                            },
                            {
                                "content": "            if not parts[0]:",
                                "type": "added"
                            },
                            {
                                "content": "                raise ValueError(\"Empty field name\")",
                                "type": "added"
                            },
                            {
                                "content": "            field, value = parts",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            field = \"tag\"",
                                "type": "added"
                            },
                            {
                                "content": "            value = parts[0]",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        if not value:",
                                "type": "added"
                            },
                            {
                                "content": "            # The `url_file` may have an empty value.",
                                "type": "added"
                            },
                            {
                                "content": "            continue",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        yield field + \"=\" + value",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def extract(stream, input_type, filename_format, url_format, meta):",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Extract and format information from `url_file`.",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    Parameters",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, filename_format, url_format):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, subpaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 64,
                            "end_lineno": 112,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "clean_meta_args",
                            "content": "def clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            field = \"tag\"\n            value = parts[0]\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 64,
                            "end_lineno": 90,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 99,
                            "length": 21
                        },
                        "new_range": {
                            "start": 128,
                            "length": 26
                        },
                        "section": "def extract(stream, input_type, filename_format, url_format):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    fmt = Formatter(colidx_to_name)",
                                "type": "context"
                            },
                            {
                                "content": "    format_filename = partial(fmt.format, filename_format)",
                                "type": "context"
                            },
                            {
                                "content": "    format_url = partial(fmt.format, url_format)",
                                "type": "context"
                            },
                            {
                                "content": "    # Unlike `filename_format` and `url_format`, `meta` is a list",
                                "type": "added"
                            },
                            {
                                "content": "    # because meta may be given multiple times on the command line.",
                                "type": "added"
                            },
                            {
                                "content": "    formats_meta = [partial(fmt.format, m) for m in meta]",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    for row in rows:",
                                "type": "context"
                            },
                            {
                                "content": "        url = format_url(row)",
                                "type": "context"
                            },
                            {
                                "content": "        filename = format_filename(row)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        subpaths = []",
                                "type": "context"
                            },
                            {
                                "content": "        if \"//\" in filename:",
                                "type": "context"
                            },
                            {
                                "content": "            for part in filename.split(\"//\")[:-1]:",
                                "type": "context"
                            },
                            {
                                "content": "                subpaths.append(os.path.join(*(subpaths + [part])))",
                                "type": "context"
                            },
                            {
                                "content": "            filename = filename.replace(\"//\", os.path.sep)",
                                "type": "context"
                            },
                            {
                                "content": "        yield filename, url, subpaths",
                                "type": "deleted"
                            },
                            {
                                "content": "        yield filename, url, meta_args, subpaths",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",",
                                "type": "context"
                            },
                            {
                                "content": "             url_format=\"{0}\", filename_format=\"{1}\",",
                                "type": "deleted"
                            },
                            {
                                "content": "             url_format=\"{0}\", filename_format=\"{1}\", meta=None,",
                                "type": "added"
                            },
                            {
                                "content": "             message=None, dry_run=False, fast=False):",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"Create and update a dataset from a list of URLs.",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, filename_format, url_format):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, subpaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 64,
                            "end_lineno": 112,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format)\n\n        if dry_run:\n            for fname, url, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        to_add = []\n        for fname, url, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            to_add.append(fname)\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(to_add, message=msg):\n            yield r",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 115,
                            "end_lineno": 289,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 149,
                            "length": 6
                        },
                        "new_range": {
                            "start": 183,
                            "length": 14
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        may contain directories.  The separator \"//\" can be used to",
                                "type": "context"
                            },
                            {
                                "content": "        indicate that the left-side directory should be created as a",
                                "type": "context"
                            },
                            {
                                "content": "        new subdataset.",
                                "type": "context"
                            },
                            {
                                "content": "    meta : str, optional",
                                "type": "added"
                            },
                            {
                                "content": "        A format string that specifies metadata.  It should be",
                                "type": "added"
                            },
                            {
                                "content": "        structured as \"<field>=<value>\".  The same placeholders from",
                                "type": "added"
                            },
                            {
                                "content": "        `url_format` can be used.  As an example, \"location={3}\" would",
                                "type": "added"
                            },
                            {
                                "content": "        mean that the value for the \"location\" metadata field should",
                                "type": "added"
                            },
                            {
                                "content": "        be set the value of the fourth column.  A plain value is",
                                "type": "added"
                            },
                            {
                                "content": "        shorthand for \"tag=<value>\".  This option can be given",
                                "type": "added"
                            },
                            {
                                "content": "        multiple times.",
                                "type": "added"
                            },
                            {
                                "content": "    message : str, optional",
                                "type": "context"
                            },
                            {
                                "content": "        Use this message when committing the URL additions.",
                                "type": "context"
                            },
                            {
                                "content": "    dry_run : bool, optional",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 149,
                            "end_lineno": 349,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 198,
                            "length": 9
                        },
                        "new_range": {
                            "start": 240,
                            "length": 12
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    from datalad.interface.results import get_status_dict",
                                "type": "context"
                            },
                            {
                                "content": "    import datalad.plugin.addurls as me",
                                "type": "context"
                            },
                            {
                                "content": "    from datalad.support.annexrepo import AnnexRepo",
                                "type": "context"
                            },
                            {
                                "content": "    from datalad.utils import assure_list",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    lgr = logging.getLogger(\"datalad.plugin.addurls\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    meta = assure_list(meta)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    if url_file is None:",
                                "type": "context"
                            },
                            {
                                "content": "        # `url_file` is not a required argument in `dlplugin` because",
                                "type": "context"
                            },
                            {
                                "content": "        # the argument before it, `dataset`, needs to be optional to",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\", meta=None,\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should\n        be set the value of the fourth column.  A plain value is\n        shorthand for \"tag=<value>\".  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.utils import assure_list\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format, meta)\n\n        if dry_run:\n            for fname, url, meta, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n                lgr.info(\"Metadata: %s\", meta)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        files_to_add = []\n        meta_to_add = []\n        for fname, url, meta, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            files_to_add.append(fname)\n            meta_to_add.append((ds_current, ds_filename, meta))\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n        for ds, fname, meta in meta_to_add:\n            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n            for arg in meta:\n                ds.repo._run_annex_command(\"metadata\",\n                                           annex_options=[\"--set\", arg, fname])\n            yield get_status_dict(action=\"addurls-metadata\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds.path, fname),\n                                  message=\"added metadata\",\n                                  status=\"ok\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 149,
                            "end_lineno": 349,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 216,
                            "length": 12
                        },
                        "new_range": {
                            "start": 261,
                            "length": 13
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        input_type = \"json\" if extension == \".json\" else \"csv\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    with open(url_file) as fd:",
                                "type": "context"
                            },
                            {
                                "content": "        info = me.extract(fd, input_type, filename_format, url_format)",
                                "type": "deleted"
                            },
                            {
                                "content": "        info = me.extract(fd, input_type, filename_format, url_format, meta)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if dry_run:",
                                "type": "context"
                            },
                            {
                                "content": "            for fname, url, _ in info:",
                                "type": "deleted"
                            },
                            {
                                "content": "            for fname, url, meta, _ in info:",
                                "type": "added"
                            },
                            {
                                "content": "                lgr.info(\"Would download %s to %s\",",
                                "type": "context"
                            },
                            {
                                "content": "                         url, os.path.join(dataset.path, fname))",
                                "type": "context"
                            },
                            {
                                "content": "                lgr.info(\"Metadata: %s\", meta)",
                                "type": "added"
                            },
                            {
                                "content": "            yield get_status_dict(action=\"addurls\",",
                                "type": "context"
                            },
                            {
                                "content": "                                  ds=dataset,",
                                "type": "context"
                            },
                            {
                                "content": "                                  status=\"ok\",",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format)\n\n        if dry_run:\n            for fname, url, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        to_add = []\n        for fname, url, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            to_add.append(fname)\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(to_add, message=msg):\n            yield r",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 115,
                            "end_lineno": 289,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 241,
                            "length": 8
                        },
                        "new_range": {
                            "start": 287,
                            "length": 9
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        annex_options = [\"--fast\"] if fast else []",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        seen_subpaths = set()",
                                "type": "context"
                            },
                            {
                                "content": "        to_add = []",
                                "type": "deleted"
                            },
                            {
                                "content": "        for fname, url, subpaths in info:",
                                "type": "deleted"
                            },
                            {
                                "content": "        files_to_add = []",
                                "type": "added"
                            },
                            {
                                "content": "        meta_to_add = []",
                                "type": "added"
                            },
                            {
                                "content": "        for fname, url, meta, subpaths in info:",
                                "type": "added"
                            },
                            {
                                "content": "            for spath in subpaths:",
                                "type": "context"
                            },
                            {
                                "content": "                if spath not in seen_subpaths:",
                                "type": "context"
                            },
                            {
                                "content": "                    if os.path.exists(os.path.join(dataset.path, spath)):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format)\n\n        if dry_run:\n            for fname, url, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        to_add = []\n        for fname, url, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            to_add.append(fname)\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(to_add, message=msg):\n            yield r",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 115,
                            "end_lineno": 289,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 277,
                            "length": 7
                        },
                        "new_range": {
                            "start": 324,
                            "length": 8
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                                                    ds_filename),",
                                "type": "context"
                            },
                            {
                                "content": "                                  status=\"ok\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "            to_add.append(fname)",
                                "type": "deleted"
                            },
                            {
                                "content": "            files_to_add.append(fname)",
                                "type": "added"
                            },
                            {
                                "content": "            meta_to_add.append((ds_current, ds_filename, meta))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        msg = message or \"\"\"\\",
                                "type": "context"
                            },
                            {
                                "content": "[DATALAD] add files from URLs",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format)\n\n        if dry_run:\n            for fname, url, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        to_add = []\n        for fname, url, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            to_add.append(fname)\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(to_add, message=msg):\n            yield r",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 115,
                            "end_lineno": 289,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 285,
                            "length": 5
                        },
                        "new_range": {
                            "start": 333,
                            "length": 17
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "url_file='{}'",
                                "type": "context"
                            },
                            {
                                "content": "url_format='{}'",
                                "type": "context"
                            },
                            {
                                "content": "filename_format='{}'\"\"\".format(url_file, url_format, filename_format)",
                                "type": "context"
                            },
                            {
                                "content": "        for r in dataset.add(to_add, message=msg):",
                                "type": "deleted"
                            },
                            {
                                "content": "        for r in dataset.add(files_to_add, message=msg):",
                                "type": "added"
                            },
                            {
                                "content": "            yield r",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        for ds, fname, meta in meta_to_add:",
                                "type": "added"
                            },
                            {
                                "content": "            lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)",
                                "type": "added"
                            },
                            {
                                "content": "            for arg in meta:",
                                "type": "added"
                            },
                            {
                                "content": "                ds.repo._run_annex_command(\"metadata\",",
                                "type": "added"
                            },
                            {
                                "content": "                                           annex_options=[\"--set\", arg, fname])",
                                "type": "added"
                            },
                            {
                                "content": "            yield get_status_dict(action=\"addurls-metadata\",",
                                "type": "added"
                            },
                            {
                                "content": "                                  ds=ds_current,",
                                "type": "added"
                            },
                            {
                                "content": "                                  type=\"file\",",
                                "type": "added"
                            },
                            {
                                "content": "                                  path=os.path.join(ds.path, fname),",
                                "type": "added"
                            },
                            {
                                "content": "                                  message=\"added metadata\",",
                                "type": "added"
                            },
                            {
                                "content": "                                  status=\"ok\")",
                                "type": "added"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format)\n\n        if dry_run:\n            for fname, url, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        to_add = []\n        for fname, url, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            to_add.append(fname)\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(to_add, message=msg):\n            yield r",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 115,
                            "end_lineno": 289,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nfrom six.moves import StringIO\n\nfrom datalad.tests.utils import assert_raises\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    fmt.format(\"{0}\", values) == \"value0\"\n    fmt.format(\"{0}\", values) == fmt.format(\"{col0}\", values)\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    fmt.format(\"{notinvals}\", values, notinvals=\"ok\") == \"ok\"\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    fnames, urls, subpaths = zip(*addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\"))\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],\n                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\"]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert list(json_output) == list(csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  list,\n                  addurls.extract(None, \"not_csv_or_json\", None, None))\n",
            "new_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nfrom six.moves import StringIO\n\nfrom datalad.tests.utils import assert_raises\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    fmt.format(\"{0}\", values) == \"value0\"\n    fmt.format(\"{0}\", values) == fmt.format(\"{col0}\", values)\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    fmt.format(\"{notinvals}\", values, notinvals=\"ok\") == \"ok\"\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"\", \"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\" atag \"], [\"tag=atag\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        assert list(addurls.clean_meta_args(args)) == expect\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    fnames, urls, meta, subpaths = zip(*addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        [\"group={age_group}\"]))\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert meta == ([\"group=kid\"], [\"group=adult\"],\n                    [\"group=adult\"], [\"group=kid\"])\n\n    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],\n                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            [\"group={age_group}\"]]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert list(json_output) == list(csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  list,\n                  addurls.extract(None, \"not_csv_or_json\", None, None, None))\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/tests/test_addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "5f77bc1aa",
                "new_commit_hash": "e1d7a6480",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/tests/test_addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/tests/test_addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 62,
                            "length": 6
                        },
                        "new_range": {
                            "start": 62,
                            "length": 18
                        },
                        "section": "def test_formatter_placeholder_nonpermitted_chars():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_clean_meta_args():",
                                "type": "added"
                            },
                            {
                                "content": "    for args, expect in [([\"\", \"field=\"], []),",
                                "type": "added"
                            },
                            {
                                "content": "                         ([\" field=yes \"], [\"field=yes\"]),",
                                "type": "added"
                            },
                            {
                                "content": "                         ([\" atag \"], [\"tag=atag\"]),",
                                "type": "added"
                            },
                            {
                                "content": "                         ([\"field= value=\"], [\"field=value=\"])]:",
                                "type": "added"
                            },
                            {
                                "content": "        assert list(addurls.clean_meta_args(args)) == expect",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    assert_raises(ValueError,",
                                "type": "added"
                            },
                            {
                                "content": "                  list,",
                                "type": "added"
                            },
                            {
                                "content": "                  addurls.clean_meta_args([\"=value\"]))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "ST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],",
                                "type": "context"
                            },
                            {
                                "content": "           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,",
                                "type": "context"
                            },
                            {
                                "content": "                     \"age_group\": \"kid\", \"now_dead\": \"no\"},",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_clean_meta_args",
                            "content": "def test_clean_meta_args():\n    for args, expect in [([\"\", \"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\" atag \"], [\"tag=atag\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        assert list(addurls.clean_meta_args(args)) == expect\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 65,
                            "end_lineno": 74,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 81,
                            "length": 16
                        },
                        "new_range": {
                            "start": 93,
                            "length": 20
                        },
                        "section": "def json_stream(data):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract():",
                                "type": "context"
                            },
                            {
                                "content": "    fnames, urls, subpaths = zip(*addurls.extract(",
                                "type": "deleted"
                            },
                            {
                                "content": "    fnames, urls, meta, subpaths = zip(*addurls.extract(",
                                "type": "added"
                            },
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{name}_{debut_season}.com\"))",
                                "type": "deleted"
                            },
                            {
                                "content": "        \"{name}_{debut_season}.com\",",
                                "type": "added"
                            },
                            {
                                "content": "        [\"group={age_group}\"]))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "                      \"adult/no/scott.csv\", \"kid/no/max.csv\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    assert meta == ([\"group=kid\"], [\"group=adult\"],",
                                "type": "added"
                            },
                            {
                                "content": "                    [\"group=adult\"], [\"group=kid\"])",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],",
                                "type": "context"
                            },
                            {
                                "content": "                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract",
                            "content": "def test_extract():\n    fnames, urls, subpaths = zip(*addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\"))\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],\n                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 83,
                            "end_lineno": 95,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 102,
                            "length": 7
                        },
                        "new_range": {
                            "start": 118,
                            "length": 8
                        },
                        "section": "def test_extract_csv_json_equal():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                    for row in ST_DATA[\"rows\"])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    args = [\"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "            \"{name}_{debut_season}.com\"]",
                                "type": "deleted"
                            },
                            {
                                "content": "            \"{name}_{debut_season}.com\",",
                                "type": "added"
                            },
                            {
                                "content": "            [\"group={age_group}\"]]",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)",
                                "type": "context"
                            },
                            {
                                "content": "    csv_output = addurls.extract(csv_rows, \"csv\", *args)",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_csv_json_equal",
                            "content": "def test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\"]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert list(json_output) == list(csv_output)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 98,
                            "end_lineno": 110,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 113,
                            "length": 4
                        },
                        "new_range": {
                            "start": 130,
                            "length": 4
                        },
                        "section": "def test_extract_csv_json_equal():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "def test_extract_wrong_input_type():",
                                "type": "context"
                            },
                            {
                                "content": "    assert_raises(ValueError,",
                                "type": "context"
                            },
                            {
                                "content": "                  list,",
                                "type": "context"
                            },
                            {
                                "content": "                  addurls.extract(None, \"not_csv_or_json\", None, None))",
                                "type": "deleted"
                            },
                            {
                                "content": "                  addurls.extract(None, \"not_csv_or_json\", None, None, None))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_wrong_input_type",
                            "content": "def test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  list,\n                  addurls.extract(None, \"not_csv_or_json\", None, None))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 113,
                            "end_lineno": 116,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "1d9f07f9154ec0510d4bdddced9d388f1f9bdf80^",
    "new_commit_hash": "1d9f07f9154ec0510d4bdddced9d388f1f9bdf80",
    "commit_message": "ENH: addurls: Set argument for setting metadata (via git-annex)",
    "commit_date": "2018-02-27T09:07:08-05:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "1d9f07f9154ec0510d4bdddced9d388f1f9bdf80",
    "test_file_codes": [
        "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nfrom six.moves import StringIO\n\nfrom datalad.tests.utils import assert_raises\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    fmt.format(\"{0}\", values) == \"value0\"\n    fmt.format(\"{0}\", values) == fmt.format(\"{col0}\", values)\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    fmt.format(\"{notinvals}\", values, notinvals=\"ok\") == \"ok\"\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"\", \"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\" atag \"], [\"tag=atag\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        assert list(addurls.clean_meta_args(args)) == expect\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    fnames, urls, meta, subpaths = zip(*addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        [\"group={age_group}\"]))\n\n    assert urls == (\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\")\n\n    assert fnames == (\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n                      \"adult/no/scott.csv\", \"kid/no/max.csv\")\n\n    assert meta == ([\"group=kid\"], [\"group=adult\"],\n                    [\"group=adult\"], [\"group=kid\"])\n\n    assert subpaths == ([\"kid\", \"kid/no\"], [\"adult\", \"adult/yes\"],\n                        [\"adult\", \"adult/no\"], [\"kid\", \"kid/no\"])\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            [\"group={age_group}\"]]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    assert list(json_output) == list(csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  list,\n                  addurls.extract(None, \"not_csv_or_json\", None, None, None))\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.9.1.dev907\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 149ms\nInstalled 25 packages in 5.48s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp9KyNiY/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp9KyNiY/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp9KyNiY/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp9KyNiY/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp9KyNiY/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpKarJIt/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpKarJIt/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpKarJIt/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpKarJIt/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpKarJIt/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_1d9f07f9154ec0510d4bdddced9d388f1f9bdf80/datalad/__init__.py\", line 17, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_1d9f07f9154ec0510d4bdddced9d388f1f9bdf80/datalad/cmd.py\", line 35, in <module>\n    from .utils import on_windows, get_tempfile_kwargs, assure_unicode\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_1d9f07f9154ec0510d4bdddced9d388f1f9bdf80/datalad/utils.py\", line 24, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 330ms\nInstalled 26 packages in 2.97s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 88 packages in 1.65s\nPrepared 1 package in 1.36s\nInstalled 77 packages in 112ms\n + appdirs==1.4.4\n + attrs==24.2.0\n + automat==22.10.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + constantly==15.1.0\n + cryptography==44.0.0\n + cssselect==1.2.0\n + datalad==0.9.1.dev1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_1d9f07f9154ec0510d4bdddced9d388f1f9bdf80)\n + deprecated==1.2.15\n + duecredit==0.9.3\n + exifread==3.0.0\n + filelock==3.12.2\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + hyperlink==21.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + incremental==22.10.0\n + iso8601==2.1.0\n + itemadapter==0.8.0\n + itemloaders==1.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jmespath==1.0.1\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack-python==0.5.6\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + parsel==1.8.1\n + patool==1.12\n + pillow==9.5.0\n + protego==0.3.0\n + pyasn1==0.5.1\n + pyasn1-modules==0.3.0\n + pycparser==2.21\n + pydicom==2.4.4\n + pydispatcher==2.0.7\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyopenssl==24.3.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + queuelib==1.6.2\n + requests==2.31.0\n + requests-file==2.1.0\n + requests-ftp==0.3.1\n + scrapy==2.9.0\n + secretstorage==3.3.3\n + service-identity==21.1.0\n + simplejson==3.19.3\n + smmap==5.0.1\n + soupsieve==2.4.1\n + tldextract==4.0.0\n + tqdm==4.67.1\n + twisted==23.8.0\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + w3lib==2.1.2\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\n + zope-interface==6.4.post2\nResolved 107 packages in 167ms\nInstalled 19 packages in 50ms\n + alabaster==0.7.13\n + babel==2.14.0\n + docutils==0.19\n + imagesize==1.4.1\n + jinja2==3.1.5\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + pygments==2.17.2\n + pypandoc==1.14\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n",
    "new_commit_res_code": 0,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_1d9f07f9154ec0510d4bdddced9d388f1f9bdf80\nplugins: cov-4.1.0\ncollected 9 items\n\nr2e_tests/test_1.py .........                                            [100%]\n\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_extract\nPASSED r2e_tests/test_1.py::test_extract_csv_json_equal\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\n============================== 9 passed in 0.66s ===============================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_1d9f07f9154ec0510d4bdddced9d388f1f9bdf80\nplugins: cov-4.1.0\ncollected 9 items\n\nr2e_tests/test_1.py .....FFFF                                            [100%]\n\n=================================== FAILURES ===================================\n_____________________________ test_clean_meta_args _____________________________\n\n    def test_clean_meta_args():\n        for args, expect in [([\"\", \"field=\"], []),\n                             ([\" field=yes \"], [\"field=yes\"]),\n                             ([\" atag \"], [\"tag=atag\"]),\n                             ([\"field= value=\"], [\"field=value=\"])]:\n>           assert list(addurls.clean_meta_args(args)) == expect\nE           AttributeError: module 'datalad.plugin.addurls' has no attribute 'clean_meta_args'\n\nr2e_tests/test_1.py:70: AttributeError\n_________________________________ test_extract _________________________________\n\n    def test_extract():\n        fnames, urls, meta, subpaths = zip(*addurls.extract(\n            json_stream(ST_DATA[\"rows\"]), \"json\",\n            \"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n>           [\"group={age_group}\"]))\nE       TypeError: extract() takes 4 positional arguments but 5 were given\n\nr2e_tests/test_1.py:100: TypeError\n_________________________ test_extract_csv_json_equal __________________________\n\n    def test_extract_csv_json_equal():\n        keys = ST_DATA[\"header\"]\n        csv_rows = [\",\".join(keys)]\n        csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                        for row in ST_DATA[\"rows\"])\n    \n        args = [\"{age_group}//{now_dead}//{name}.csv\",\n                \"{name}_{debut_season}.com\",\n                [\"group={age_group}\"]]\n    \n>       json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\nE       TypeError: extract() takes 4 positional arguments but 5 were given\n\nr2e_tests/test_1.py:124: TypeError\n________________________ test_extract_wrong_input_type _________________________\n\n    def test_extract_wrong_input_type():\n        assert_raises(ValueError,\n                      list,\n>                     addurls.extract(None, \"not_csv_or_json\", None, None, None))\nE       TypeError: extract() takes 4 positional arguments but 5 were given\n\nr2e_tests/test_1.py:133: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nFAILED r2e_tests/test_1.py::test_clean_meta_args - AttributeError: module 'da...\nFAILED r2e_tests/test_1.py::test_extract - TypeError: extract() takes 4 posit...\nFAILED r2e_tests/test_1.py::test_extract_csv_json_equal - TypeError: extract(...\nFAILED r2e_tests/test_1.py::test_extract_wrong_input_type - TypeError: extrac...\n========================= 4 failed, 5 passed in 0.42s ==========================\n",
    "old_commit_res_stderr": ""
}
datalad/plugin/addurls.py datalad/plugin/tests/test_addurls.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]

**Title:** Metadata Arguments Not Handled Correctly in `addurls` Plugin Causing Test Failures

**Description:**
When using the `addurls` plugin to set metadata, the functions responsible for processing metadata arguments are either missing or not accepting the required parameters. This leads to errors during testing, preventing proper handling of metadata.

**Buggy Code Example:**
```python
def test_extract():
    fnames, urls, meta, subpaths = zip(*addurls.extract(
        json_stream(ST_DATA['rows']), 'json',
        '{age_group}//{now_dead}//{name}.csv',
        '{name}_{debut_season}.com',
        ['group={age_group}']))
```

**Expected Behavior:**
The `extract` function should correctly accept and process metadata arguments, allowing tests to pass without errors. Specifically, the `clean_meta_args` function should be available for processing metadata, and the `extract` function should handle the additional `meta` parameter.

**Actual Behavior:**
- **Error 1:** 
  ```
  AttributeError: module 'datalad.plugin.addurls' has no attribute 'clean_meta_args'
  ```
  This error occurs because the `clean_meta_args` function is missing from the `addurls` module.

- **Error 2:** 
  ```
  TypeError: extract() takes 4 positional arguments but 5 were given
  ```
  This error arises because the `extract` function is being called with an extra `meta` argument that it does not accept.

These issues prevent the proper setting of metadata and cause related tests to fail.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import string

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    """

    def __init__(self, idx_to_name, *args, **kwargs):
        self.idx_to_name = idx_to_name or {}
        super(Formatter, self).__init__(*args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]
        if not isinstance(data, Mapping):
            raise ValueError("First positional argument should be mapping")

        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            return data[self.idx_to_name[key_int]]

        try:
            return data[key]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)


def extract(stream, input_type, filename_format, url_format):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.

    All other parameters match those described in `dlplugin`.

    Returns
    -------
    A generator that, for each item in `url_file`, yields a tuple that
    contains the formatted filename (with any "//" collapsed to a
    single separator), the formatted url, and a list of subdataset
    paths.  The length of the subdataset paths list will be equal to
    the number of "//" occurrences in the `filename_format`.
    """
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        colidx_to_name = dict(enumerate(headers))
        rows = (dict(zip(headers, r)) for r in csvrows)
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        colidx_to_name = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")

    fmt = Formatter(colidx_to_name)
    format_filename = partial(fmt.format, filename_format)
    format_url = partial(fmt.format, url_format)

    for row in rows:
        url = format_url(row)
        filename = format_filename(row)

        subpaths = []
        if "//" in filename:
            for part in filename.split("//")[:-1]:
                subpaths.append(os.path.join(*(subpaths + [part])))
            filename = filename.replace("//", os.path.sep)
        yield filename, url, subpaths


def dlplugin(dataset=None, url_file=None, input_type="ext",
             url_format="{0}", filename_format="{1}",
             message=None, dry_run=False, fast=False):
    """Create and update a dataset from a list of URLs.

    Parameters
    ----------
    dataset : Dataset
        Add the URLs to this dataset (or possibly subdatasets of this
        dataset).  An empty or non-existent directory is passed to
        create a new dataset.  New subdatasets can be specified with
        `filename_format`.
    url_file : str
        A file that contains URLs or information that can be used to
        construct URLs.  Depending on the value of `input_type`, this
        should be a CSV file (with a header as the first row) or a
        JSON file (structured as a list of objects with string
        values).
    input_type : {"ext", "csv", "json"}, optional
        Whether `url_file` should be considered a CSV file or a JSON
        file.  The default value, "ext", means to consider `url_file`
        as a JSON file if it ends with ".json".  Otherwise, treat it
        as a CSV file.
    url_format : str, optional
        A format string that specifies the URL for each entry.  This
        value is similar to a normal Python format string where the
        names from `url_file` (column names for a CSV or properties
        for JSON) are available as placeholders.  If `url_file` is a
        CSV file, a positional index can also be used (i.e., "{0}" for
        the first column).  Note that a placeholder cannot contain a
        ':' or '!'.
    filename_format : str, optional
        Like `url_format`, but this format string specifies the file
        to which the URL's content will be downloaded.  The file name
        may contain directories.  The separator "//" can be used to
        indicate that the left-side directory should be created as a
        new subdataset.
    message : str, optional
        Use this message when committing the URL additions.
    dry_run : bool, optional
        Report which URLs would be downloaded to which files and then
        exit.
    fast : bool, optional
        If True, add the URLs, but don't download their content.
        Underneath, this passes the --fast flag to `git annex addurl`.

    Examples
    --------
    Consider a file "avatars.csv" that contains

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and
    'ext' fields, we could run

        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv
          url_format='{link}' filename_format='{who}.{ext}' fast=True

    The '-d avatar_ds' is used to create a new dataset in
    "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new
    subdataset in an "avatars" subdirectory, we could use "//" in the
    `filename_format` argument:

        $ datalad plugin addurls url_file=avatars.csv
          url_format='{link}' filename_format='avatars//{who}.{ext}'
          fast=True

    Note
    ----
    For users familiar with 'git annex addurl': A large part of this
    plugin's functionality can be viewed as transforming data from
    `url_file` into a "url filename" format that fed to 'git annex
    addurl --batch --with-files'.
    """
    from itertools import dropwhile
    import logging
    import os

    from datalad.distribution.dataset import Dataset
    from datalad.interface.results import get_status_dict
    import datalad.plugin.addurls as me
    from datalad.support.annexrepo import AnnexRepo

    lgr = logging.getLogger("datalad.plugin.addurls")

    if url_file is None:
        # `url_file` is not a required argument in `dlplugin` because
        # the argument before it, `dataset`, needs to be optional to
        # support the creation of new datasets.
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="Must specify url_file argument")
        return

    if input_type == "ext":
        extension = os.path.splitext(url_file)[1]
        input_type = "json" if extension == ".json" else "csv"

    with open(url_file) as fd:
        info = me.extract(fd, input_type, filename_format, url_format)

        if dry_run:
            for fname, url, _ in info:
                lgr.info("Would download %s to %s",
                         url, os.path.join(dataset.path, fname))
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="ok",
                                  message="dry-run finished")
            return

        if not dataset.repo:
            # Populate a new dataset with the URLs.
            dataset.create()
        elif not isinstance(dataset.repo, AnnexRepo):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message="not an annex repo")
            return

        annex_options = ["--fast"] if fast else []

        seen_subpaths = set()
        to_add = []
        for fname, url, subpaths in info:
            for spath in subpaths:
                if spath not in seen_subpaths:
                    if os.path.exists(os.path.join(dataset.path, spath)):
                        lgr.warning(
                            "Not creating subdataset at existing path: %s",
                            spath)
                    else:
                        dataset.create(spath)
                seen_subpaths.add(spath)

            if subpaths:
                # Adjust the dataset and filename for an `addurl` call
                # from within the subdataset that will actually contain
                # the link.
                datasets = [Dataset(os.path.join(dataset.path, sp))
                            for sp in subpaths]
                ds_current = next(dropwhile(lambda d: not d.repo,
                                            reversed(datasets)))
                ds_filename = os.path.relpath(
                    os.path.join(dataset.path, fname),
                    ds_current.path)
            else:
                ds_current = dataset
                ds_filename = fname

            ds_current.repo.add_url_to_file(ds_filename, url,
                                            batch=True, options=annex_options)
            yield get_status_dict(action="addurls",
                                  ds=ds_current,
                                  type="file",
                                  path=os.path.join(ds_current.path,
                                                    ds_filename),
                                  status="ok")

            to_add.append(fname)

        msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)
        for r in dataset.add(to_add, message=msg):
            yield r

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]

**Title:** Metadata Arguments Not Handled Correctly in `addurls` Plugin Causing Test Failures

**Description:**
When using the `addurls` plugin to set metadata, the functions responsible for processing metadata arguments are either missing or not accepting the required parameters. This leads to errors during testing, preventing proper handling of metadata.

**Buggy Code Example:**
```python
def test_extract():
    fnames, urls, meta, subpaths = zip(*addurls.extract(
        json_stream(ST_DATA['rows']), 'json',
        '{age_group}//{now_dead}//{name}.csv',
        '{name}_{debut_season}.com',
        ['group={age_group}']))
```

**Expected Behavior:**
The `extract` function should correctly accept and process metadata arguments, allowing tests to pass without errors. Specifically, the `clean_meta_args` function should be available for processing metadata, and the `extract` function should handle the additional `meta` parameter.

**Actual Behavior:**
- **Error 1:** 
  ```
  AttributeError: module 'datalad.plugin.addurls' has no attribute 'clean_meta_args'
  ```
  This error occurs because the `clean_meta_args` function is missing from the `addurls` module.

- **Error 2:** 
  ```
  TypeError: extract() takes 4 positional arguments but 5 were given
  ```
  This error arises because the `extract` function is being called with an extra `meta` argument that it does not accept.

These issues prevent the proper setting of metadata and cause related tests to fail.

[/ISSUE]
{
    "test_formatter": "PASSED",
    "test_formatter_no_idx_map": "PASSED",
    "test_formatter_no_mapping_arg": "PASSED",
    "test_formatter_placeholder_with_spaces": "PASSED",
    "test_formatter_placeholder_nonpermitted_chars": "PASSED",
    "test_clean_meta_args": "PASSED",
    "test_extract": "PASSED",
    "test_extract_csv_json_equal": "PASSED",
    "test_extract_wrong_input_type": "PASSED"
}
diff --git a/datalad/plugin/addurls.py b/datalad/plugin/addurls.py
index a10d6f1e7..c9d8b3fe4 100644
--- a/datalad/plugin/addurls.py
+++ b/datalad/plugin/addurls.py
@@ -61,7 +61,36 @@ class Formatter(string.Formatter):
                 key, args, kwargs)
 
 
-def extract(stream, input_type, filename_format, url_format):
+def clean_meta_args(args):
+    """Prepare formatted metadata arguments to be passed to git-annex.
+
+    Parameters
+    ----------
+    args : iterable of str
+        Formatted metadata arguments for 'git-annex metadata --set'.
+
+    Returns
+    -------
+    Generator that yields processed arguments (str).
+    """
+    for arg in args:
+        parts = [x.strip() for x in arg.split("=", 1)]
+        if len(parts) == 2:
+            if not parts[0]:
+                raise ValueError("Empty field name")
+            field, value = parts
+        else:
+            field = "tag"
+            value = parts[0]
+
+        if not value:
+            # The `url_file` may have an empty value.
+            continue
+
+        yield field + "=" + value
+
+
+def extract(stream, input_type, filename_format, url_format, meta):
     """Extract and format information from `url_file`.
 
     Parameters
@@ -99,21 +128,26 @@ def extract(stream, input_type, filename_format, url_format):
     fmt = Formatter(colidx_to_name)
     format_filename = partial(fmt.format, filename_format)
     format_url = partial(fmt.format, url_format)
+    # Unlike `filename_format` and `url_format`, `meta` is a list
+    # because meta may be given multiple times on the command line.
+    formats_meta = [partial(fmt.format, m) for m in meta]
 
     for row in rows:
         url = format_url(row)
         filename = format_filename(row)
 
+        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))
+
         subpaths = []
         if "//" in filename:
             for part in filename.split("//")[:-1]:
                 subpaths.append(os.path.join(*(subpaths + [part])))
             filename = filename.replace("//", os.path.sep)
-        yield filename, url, subpaths
+        yield filename, url, meta_args, subpaths
 
 
 def dlplugin(dataset=None, url_file=None, input_type="ext",
-             url_format="{0}", filename_format="{1}",
+             url_format="{0}", filename_format="{1}", meta=None,
              message=None, dry_run=False, fast=False):
     """Create and update a dataset from a list of URLs.
 
@@ -149,6 +183,14 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
         may contain directories.  The separator "//" can be used to
         indicate that the left-side directory should be created as a
         new subdataset.
+    meta : str, optional
+        A format string that specifies metadata.  It should be
+        structured as "<field>=<value>".  The same placeholders from
+        `url_format` can be used.  As an example, "location={3}" would
+        mean that the value for the "location" metadata field should
+        be set the value of the fourth column.  A plain value is
+        shorthand for "tag=<value>".  This option can be given
+        multiple times.
     message : str, optional
         Use this message when committing the URL additions.
     dry_run : bool, optional
@@ -198,9 +240,12 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
     from datalad.interface.results import get_status_dict
     import datalad.plugin.addurls as me
     from datalad.support.annexrepo import AnnexRepo
+    from datalad.utils import assure_list
 
     lgr = logging.getLogger("datalad.plugin.addurls")
 
+    meta = assure_list(meta)
+
     if url_file is None:
         # `url_file` is not a required argument in `dlplugin` because
         # the argument before it, `dataset`, needs to be optional to
@@ -216,12 +261,13 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
         input_type = "json" if extension == ".json" else "csv"
 
     with open(url_file) as fd:
-        info = me.extract(fd, input_type, filename_format, url_format)
+        info = me.extract(fd, input_type, filename_format, url_format, meta)
 
         if dry_run:
-            for fname, url, _ in info:
+            for fname, url, meta, _ in info:
                 lgr.info("Would download %s to %s",
                          url, os.path.join(dataset.path, fname))
+                lgr.info("Metadata: %s", meta)
             yield get_status_dict(action="addurls",
                                   ds=dataset,
                                   status="ok",
@@ -241,8 +287,9 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
         annex_options = ["--fast"] if fast else []
 
         seen_subpaths = set()
-        to_add = []
-        for fname, url, subpaths in info:
+        files_to_add = []
+        meta_to_add = []
+        for fname, url, meta, subpaths in info:
             for spath in subpaths:
                 if spath not in seen_subpaths:
                     if os.path.exists(os.path.join(dataset.path, spath)):
@@ -277,7 +324,8 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
                                                     ds_filename),
                                   status="ok")
 
-            to_add.append(fname)
+            files_to_add.append(fname)
+            meta_to_add.append((ds_current, ds_filename, meta))
 
         msg = message or """\
 [DATALAD] add files from URLs
@@ -285,5 +333,17 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
 url_file='{}'
 url_format='{}'
 filename_format='{}'""".format(url_file, url_format, filename_format)
-        for r in dataset.add(to_add, message=msg):
+        for r in dataset.add(files_to_add, message=msg):
             yield r
+
+        for ds, fname, meta in meta_to_add:
+            lgr.debug("Adding metadata to %s in %s", fname, ds.path)
+            for arg in meta:
+                ds.repo._run_annex_command("metadata",
+                                           annex_options=["--set", arg, fname])
+            yield get_status_dict(action="addurls-metadata",
+                                  ds=ds_current,
+                                  type="file",
+                                  path=os.path.join(ds.path, fname),
+                                  message="added metadata",
+                                  status="ok")

{"datalad/plugin/addurls.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport string\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    \"\"\"\n\n    def __init__(self, idx_to_name, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n        if not isinstance(data, Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            return data[self.idx_to_name[key_int]]\n\n        try:\n            return data[key]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n\ndef extract(stream, input_type, filename_format, url_format):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A generator that, for each item in `url_file`, yields a tuple that\n    contains the formatted filename (with any \"//\" collapsed to a\n    single separator), the formatted url, and a list of subdataset\n    paths.  The length of the subdataset paths list will be equal to\n    the number of \"//\" occurrences in the `filename_format`.\n    \"\"\"\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        colidx_to_name = dict(enumerate(headers))\n        rows = (dict(zip(headers, r)) for r in csvrows)\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        colidx_to_name = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n\n    fmt = Formatter(colidx_to_name)\n    format_filename = partial(fmt.format, filename_format)\n    format_url = partial(fmt.format, url_format)\n\n    for row in rows:\n        url = format_url(row)\n        filename = format_filename(row)\n\n        subpaths = []\n        if \"//\" in filename:\n            for part in filename.split(\"//\")[:-1]:\n                subpaths.append(os.path.join(*(subpaths + [part])))\n            filename = filename.replace(\"//\", os.path.sep)\n        yield filename, url, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             message=None, dry_run=False, fast=False):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file\n        to which the URL's content will be downloaded.  The file name\n        may contain directories.  The separator \"//\" can be used to\n        indicate that the left-side directory should be created as a\n        new subdataset.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    from itertools import dropwhile\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        info = me.extract(fd, input_type, filename_format, url_format)\n\n        if dry_run:\n            for fname, url, _ in info:\n                lgr.info(\"Would download %s to %s\",\n                         url, os.path.join(dataset.path, fname))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            dataset.create()\n        elif not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        annex_options = [\"--fast\"] if fast else []\n\n        seen_subpaths = set()\n        to_add = []\n        for fname, url, subpaths in info:\n            for spath in subpaths:\n                if spath not in seen_subpaths:\n                    if os.path.exists(os.path.join(dataset.path, spath)):\n                        lgr.warning(\n                            \"Not creating subdataset at existing path: %s\",\n                            spath)\n                    else:\n                        dataset.create(spath)\n                seen_subpaths.add(spath)\n\n            if subpaths:\n                # Adjust the dataset and filename for an `addurl` call\n                # from within the subdataset that will actually contain\n                # the link.\n                datasets = [Dataset(os.path.join(dataset.path, sp))\n                            for sp in subpaths]\n                ds_current = next(dropwhile(lambda d: not d.repo,\n                                            reversed(datasets)))\n                ds_filename = os.path.relpath(\n                    os.path.join(dataset.path, fname),\n                    ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = fname\n\n            ds_current.repo.add_url_to_file(ds_filename, url,\n                                            batch=True, options=annex_options)\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds_current,\n                                  type=\"file\",\n                                  path=os.path.join(ds_current.path,\n                                                    ds_filename),\n                                  status=\"ok\")\n\n            to_add.append(fname)\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n        for r in dataset.add(to_add, message=msg):\n            yield r\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]

**Title:** Metadata Arguments Not Handled Correctly in `addurls` Plugin Causing Test Failures

**Description:**
When using the `addurls` plugin to set metadata, the functions responsible for processing metadata arguments are either missing or not accepting the required parameters. This leads to errors during testing, preventing proper handling of metadata.

**Buggy Code Example:**
```python
def test_extract():
    fnames, urls, meta, subpaths = zip(*addurls.extract(
        json_stream(ST_DATA['rows']), 'json',
        '{age_group}//{now_dead}//{name}.csv',
        '{name}_{debut_season}.com',
        ['group={age_group}']))
```

**Expected Behavior:**
The `extract` function should correctly accept and process metadata arguments, allowing tests to pass without errors. Specifically, the `clean_meta_args` function should be available for processing metadata, and the `extract` function should handle the additional `meta` parameter.

**Actual Behavior:**
- **Error 1:** 
  ```
  AttributeError: module 'datalad.plugin.addurls' has no attribute 'clean_meta_args'
  ```
  This error occurs because the `clean_meta_args` function is missing from the `addurls` module.

- **Error 2:** 
  ```
  TypeError: extract() takes 4 positional arguments but 5 were given
  ```
  This error arises because the `extract` function is being called with an extra `meta` argument that it does not accept.

These issues prevent the proper setting of metadata and cause related tests to fail.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import string

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    """

    def __init__(self, idx_to_name, *args, **kwargs):
        self.idx_to_name = idx_to_name or {}
        super(Formatter, self).__init__(*args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]
        if not isinstance(data, Mapping):
            raise ValueError("First positional argument should be mapping")

        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            return data[self.idx_to_name[key_int]]

        try:
            return data[key]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)


def extract(stream, input_type, filename_format, url_format):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.

    All other parameters match those described in `dlplugin`.

    Returns
    -------
    A generator that, for each item in `url_file`, yields a tuple that
    contains the formatted filename (with any "//" collapsed to a
    single separator), the formatted url, and a list of subdataset
    paths.  The length of the subdataset paths list will be equal to
    the number of "//" occurrences in the `filename_format`.
    """
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        colidx_to_name = dict(enumerate(headers))
        rows = (dict(zip(headers, r)) for r in csvrows)
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        colidx_to_name = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")

    fmt = Formatter(colidx_to_name)
    format_filename = partial(fmt.format, filename_format)
    format_url = partial(fmt.format, url_format)

    for row in rows:
        url = format_url(row)
        filename = format_filename(row)

        subpaths = []
        if "//" in filename:
            for part in filename.split("//")[:-1]:
                subpaths.append(os.path.join(*(subpaths + [part])))
            filename = filename.replace("//", os.path.sep)
        yield filename, url, subpaths


def dlplugin(dataset=None, url_file=None, input_type="ext",
             url_format="{0}", filename_format="{1}",
             message=None, dry_run=False, fast=False):
    """Create and update a dataset from a list of URLs.

    Parameters
    ----------
    dataset : Dataset
        Add the URLs to this dataset (or possibly subdatasets of this
        dataset).  An empty or non-existent directory is passed to
        create a new dataset.  New subdatasets can be specified with
        `filename_format`.
    url_file : str
        A file that contains URLs or information that can be used to
        construct URLs.  Depending on the value of `input_type`, this
        should be a CSV file (with a header as the first row) or a
        JSON file (structured as a list of objects with string
        values).
    input_type : {"ext", "csv", "json"}, optional
        Whether `url_file` should be considered a CSV file or a JSON
        file.  The default value, "ext", means to consider `url_file`
        as a JSON file if it ends with ".json".  Otherwise, treat it
        as a CSV file.
    url_format : str, optional
        A format string that specifies the URL for each entry.  This
        value is similar to a normal Python format string where the
        names from `url_file` (column names for a CSV or properties
        for JSON) are available as placeholders.  If `url_file` is a
        CSV file, a positional index can also be used (i.e., "{0}" for
        the first column).  Note that a placeholder cannot contain a
        ':' or '!'.
    filename_format : str, optional
        Like `url_format`, but this format string specifies the file
        to which the URL's content will be downloaded.  The file name
        may contain directories.  The separator "//" can be used to
        indicate that the left-side directory should be created as a
        new subdataset.
    message : str, optional
        Use this message when committing the URL additions.
    dry_run : bool, optional
        Report which URLs would be downloaded to which files and then
        exit.
    fast : bool, optional
        If True, add the URLs, but don't download their content.
        Underneath, this passes the --fast flag to `git annex addurl`.

    Examples
    --------
    Consider a file "avatars.csv" that contains

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and
    'ext' fields, we could run

        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv
          url_format='{link}' filename_format='{who}.{ext}' fast=True

    The '-d avatar_ds' is used to create a new dataset in
    "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new
    subdataset in an "avatars" subdirectory, we could use "//" in the
    `filename_format` argument:

        $ datalad plugin addurls url_file=avatars.csv
          url_format='{link}' filename_format='avatars//{who}.{ext}'
          fast=True

    Note
    ----
    For users familiar with 'git annex addurl': A large part of this
    plugin's functionality can be viewed as transforming data from
    `url_file` into a "url filename" format that fed to 'git annex
    addurl --batch --with-files'.
    """
    from itertools import dropwhile
    import logging
    import os

    from datalad.distribution.dataset import Dataset
    from datalad.interface.results import get_status_dict
    import datalad.plugin.addurls as me
    from datalad.support.annexrepo import AnnexRepo

    lgr = logging.getLogger("datalad.plugin.addurls")

    if url_file is None:
        # `url_file` is not a required argument in `dlplugin` because
        # the argument before it, `dataset`, needs to be optional to
        # support the creation of new datasets.
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="Must specify url_file argument")
        return

    if input_type == "ext":
        extension = os.path.splitext(url_file)[1]
        input_type = "json" if extension == ".json" else "csv"

    with open(url_file) as fd:
        info = me.extract(fd, input_type, filename_format, url_format)

        if dry_run:
            for fname, url, _ in info:
                lgr.info("Would download %s to %s",
                         url, os.path.join(dataset.path, fname))
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="ok",
                                  message="dry-run finished")
            return

        if not dataset.repo:
            # Populate a new dataset with the URLs.
            dataset.create()
        elif not isinstance(dataset.repo, AnnexRepo):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message="not an annex repo")
            return

        annex_options = ["--fast"] if fast else []

        seen_subpaths = set()
        to_add = []
        for fname, url, subpaths in info:
            for spath in subpaths:
                if spath not in seen_subpaths:
                    if os.path.exists(os.path.join(dataset.path, spath)):
                        lgr.warning(
                            "Not creating subdataset at existing path: %s",
                            spath)
                    else:
                        dataset.create(spath)
                seen_subpaths.add(spath)

            if subpaths:
                # Adjust the dataset and filename for an `addurl` call
                # from within the subdataset that will actually contain
                # the link.
                datasets = [Dataset(os.path.join(dataset.path, sp))
                            for sp in subpaths]
                ds_current = next(dropwhile(lambda d: not d.repo,
                                            reversed(datasets)))
                ds_filename = os.path.relpath(
                    os.path.join(dataset.path, fname),
                    ds_current.path)
            else:
                ds_current = dataset
                ds_filename = fname

            ds_current.repo.add_url_to_file(ds_filename, url,
                                            batch=True, options=annex_options)
            yield get_status_dict(action="addurls",
                                  ds=ds_current,
                                  type="file",
                                  path=os.path.join(ds_current.path,
                                                    ds_filename),
                                  status="ok")

            to_add.append(fname)

        msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)
        for r in dataset.add(to_add, message=msg):
            yield r

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user