{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport re\nimport string\n\nfrom six import string_types\nfrom six.moves.urllib.parse import urlparse\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name, missing_value=None, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n\n\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, string_types):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(*(spaths + [part]))\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    legal = re.compile(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\")\n    return True if legal.match(name) else False\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\ndef _read(stream, input_type):\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n    return rows, idx_map\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        filename = format_fn(row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_url_names(url):\n    \"\"\"Assign a name to each part of a URL's path.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys '_url0' through '_urlN' for a path with N+1\n    components.  There is also a `_url_basename` for the rightmost\n    entry.\n    \"\"\"\n    path = urlparse(url).path.strip(\"/\")\n    if not path:\n        return {}\n\n    url_parts = path.split(\"/\")\n    names = {}\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    names[\"_url_basename\"] = url_parts[-1]\n    return names\n\n\ndef extract(stream, input_type, filename_format, url_format,\n            no_autometa, meta, missing_value):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if not no_autometa:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        #\n        # TODO: Add a command line argument for excluding a particular\n        # column(s), maybe based on a regexp.\n        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())\n                                          if c != urlcol)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    infos = []\n    for row in rows:\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n        infos.append({\"url\": format_url(row),\n                      \"meta_args\": meta_args})\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    if any(i.startswith(\"_url\") for i in get_fmt_names(filename_format)):\n        for row, info in zip(rows, infos):\n            row.update(get_url_names(info[\"url\"]))\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows, infos)\n    return infos, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             no_autometa=False, meta=None,\n             message=None, dry_run=False, fast=False,\n             ifexists=None, missing_value=None):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file to\n        which the URL's content will be downloaded.  The file name may\n        contain directories.  The separator \"//\" can be used to indicate\n        that the left-side directory should be created as a new\n        subdataset.\n\n        In addition to the placeholders described in `url_format`, there\n        are a few special placeholder.\n\n          - \"_repindex\"\n\n            The constructed file names must be unique across all fields\n            rows.  To avoid collisions, the special placeholder\n            \"_repindex\" can added to the formatter.  It value will start\n            at 0 and increment every time a file name repeats.\n\n          - \"_urlN\"\n\n            Each part of the formatted URL is available.  For example,\n            in \"http://datalad.org/for/git-users\", \"_url0\" and \"_url1\"\n            would map to \"for\" and \"git-users\", respectively.\n\n            The final part is also available as \"_url_basename\".\n    no_autometa : bool, optional\n        By default, metadata field=value pairs are constructed with each\n        column in `url_file`, excluding any single column that is\n        specified via `url_format`.  Set this flag to True to disable\n        this behavior.  Metadata can still be set explicitly with the\n        `meta` argument.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should be\n        set the value of the fourth column.  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n    ifexists : {None, 'overwrite', 'skip'}\n        What to do if a constructed file name already exists.  The\n        default (None) behavior to proceed with the `git annex addurl`,\n        which will failed if the file size has changed.  If set to\n        'overwrite', remove the old file before adding the new one.  If\n        set to 'skip', do not add the new file.\n    missing_value : str, optional\n        When an empty string is encountered, use this value instead.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import AnnexBatchCommandError\n    from datalad.utils import assure_list\n    from datalad.ui import ui\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"not an annex repo\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        rows, subpaths = me.extract(fd, input_type,\n                                    filename_format, url_format,\n                                    no_autometa, meta,\n                                    missing_value)\n\n    all_files = [row[\"filename\"] for row in rows]\n    if len(all_files) != len(set(all_files)):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=(\"There are file name collisions; \"\n                                       \"consider using {_repindex}\"))\n        return\n\n    if dry_run:\n        for subpath in subpaths:\n            lgr.info(\"Would create a subdataset at %s\", subpath)\n        for row in rows:\n            lgr.info(\"Would download %s to %s\",\n                     row[\"url\"], os.path.join(dataset.path, row[\"filename\"]))\n            lgr.info(\"Metadata: %s\", row[\"meta_args\"])\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"ok\",\n                              message=\"dry-run finished\")\n        return\n\n    if not dataset.repo:\n        # Populate a new dataset with the URLs.\n        for r in dataset.create(result_xfm=None, return_type='generator'):\n            yield r\n\n    annex_options = [\"--fast\"] if fast else []\n\n    for spath in subpaths:\n        if os.path.exists(os.path.join(dataset.path, spath)):\n            lgr.warning(\n                \"Not creating subdataset at existing path: %s\",\n                spath)\n        else:\n            for r in dataset.create(spath, result_xfm=None,\n                                    return_type='generator'):\n                yield r\n\n    pbar_addurl = ui.get_progressbar(total=len(rows), label=\"Adding files\",\n                                     unit=\" Files\")\n    pbar_addurl.start()\n\n    files_to_add = []\n    meta_to_add = []\n    addurl_results = []\n    for row_idx, row in enumerate(rows, 1):\n        pbar_addurl.update(row_idx)\n        fname_abs = os.path.join(dataset.path, row[\"filename\"])\n        if row[\"subpath\"]:\n            # Adjust the dataset and filename for an `addurl` call\n            # from within the subdataset that will actually contain\n            # the link.\n            ds_current = Dataset(os.path.join(dataset.path, row[\"subpath\"]))\n            ds_filename = os.path.relpath(fname_abs, ds_current.path)\n        else:\n            ds_current = dataset\n            ds_filename = row[\"filename\"]\n\n        if os.path.exists(fname_abs) or os.path.islink(fname_abs):\n            if ifexists == \"skip\":\n                addurl_results.append(\n                    get_status_dict(action=\"addurls\",\n                                    ds=ds_current,\n                                    type=\"file\",\n                                    path=fname_abs,\n                                    status=\"notneeded\"))\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", fname_abs)\n                os.unlink(fname_abs)\n            else:\n                lgr.debug(\"File %s already exists\", fname_abs)\n\n        try:\n            ds_current.repo.add_url_to_file(ds_filename, row[\"url\"],\n                                            batch=True, options=annex_options)\n        except AnnexBatchCommandError:\n            addurl_results.append(\n                get_status_dict(action=\"addurls\",\n                                ds=ds_current,\n                                type=\"file\",\n                                path=os.path.join(ds_current.path,\n                                                  ds_filename),\n                                message=\"failed to add URL\",\n                                status=\"error\"))\n\n            continue\n        # Collect the status dicts to yield later so that we don't\n        # interrupt the progress bar.\n        addurl_results.append(\n            get_status_dict(action=\"addurls\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds_current.path,\n                                              ds_filename),\n                            status=\"ok\"))\n\n        files_to_add.append(row[\"filename\"])\n        meta_to_add.append((ds_current, ds_filename, row[\"meta_args\"]))\n    pbar_addurl.finish()\n\n    for result in addurl_results:\n        yield result\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n    if files_to_add:\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n    # TODO: Wrap this repeated \"delayed results/progress bar\" pattern.\n    pbar_meta = ui.get_progressbar(total=len(meta_to_add),\n                                   label=\"Adding metadata\", unit=\" Files\")\n    pbar_meta.start()\n\n    meta_results = []\n    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):\n        pbar_meta.update(meta_idx)\n        lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n        for arg in meta:\n            ds.repo._run_annex_command(\"metadata\",\n                                       annex_options=[\"--set\", arg, fname])\n        meta_results.append(\n            get_status_dict(action=\"addurls-metadata\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds.path, fname),\n                            message=\"added metadata\",\n                            status=\"ok\"))\n\n    for result in meta_results:\n        yield result\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport re\nimport string\n\nfrom six import string_types\nfrom six.moves.urllib.parse import urlparse\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name, missing_value=None, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n\n\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, string_types):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(*(spaths + [part]))\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    legal = re.compile(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\")\n    return True if legal.match(name) else False\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\ndef _read(stream, input_type):\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n    return rows, idx_map\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        filename = format_fn(row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_url_names(url):\n    \"\"\"Assign a name to each part of a URL's path.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys '_url0' through '_urlN' for a path with N+1\n    components.  There is also a `_url_basename` for the rightmost\n    entry.\n    \"\"\"\n    path = urlparse(url).path.strip(\"/\")\n    if not path:\n        return {}\n\n    url_parts = path.split(\"/\")\n    names = {}\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    names[\"_url_basename\"] = url_parts[-1]\n    return names\n\n\ndef extract(stream, input_type, filename_format, url_format,\n            exclude_autometa, meta, missing_value):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    infos = []\n    for row in rows:\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n        infos.append({\"url\": format_url(row),\n                      \"meta_args\": meta_args})\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    if any(i.startswith(\"_url\") for i in get_fmt_names(filename_format)):\n        for row, info in zip(rows, infos):\n            row.update(get_url_names(info[\"url\"]))\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows, infos)\n    return infos, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             exclude_autometa=None, meta=None,\n             message=None, dry_run=False, fast=False,\n             ifexists=None, missing_value=None):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file to\n        which the URL's content will be downloaded.  The file name may\n        contain directories.  The separator \"//\" can be used to indicate\n        that the left-side directory should be created as a new\n        subdataset.\n\n        In addition to the placeholders described in `url_format`, there\n        are a few special placeholder.\n\n          - \"_repindex\"\n\n            The constructed file names must be unique across all fields\n            rows.  To avoid collisions, the special placeholder\n            \"_repindex\" can added to the formatter.  It value will start\n            at 0 and increment every time a file name repeats.\n\n          - \"_urlN\"\n\n            Each part of the formatted URL is available.  For example,\n            in \"http://datalad.org/for/git-users\", \"_url0\" and \"_url1\"\n            would map to \"for\" and \"git-users\", respectively.\n\n            The final part is also available as \"_url_basename\".\n    exclude_autometa : str, optional\n        By default, metadata field=value pairs are constructed with each\n        column in `url_file`, excluding any single column that is\n        specified via `url_format`.  This argument can be used to\n        exclude columns that match a regular expression.  If set to '*'\n        or an empty string, automatic metadata extraction is disabled\n        completely.  This argument does not affect metadata set\n        explicitly with the `meta` argument.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should be\n        set the value of the fourth column.  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n    ifexists : {None, 'overwrite', 'skip'}\n        What to do if a constructed file name already exists.  The\n        default (None) behavior to proceed with the `git annex addurl`,\n        which will failed if the file size has changed.  If set to\n        'overwrite', remove the old file before adding the new one.  If\n        set to 'skip', do not add the new file.\n    missing_value : str, optional\n        When an empty string is encountered, use this value instead.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import AnnexBatchCommandError\n    from datalad.utils import assure_list\n    from datalad.ui import ui\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"not an annex repo\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        rows, subpaths = me.extract(fd, input_type,\n                                    filename_format, url_format,\n                                    exclude_autometa, meta,\n                                    missing_value)\n\n    all_files = [row[\"filename\"] for row in rows]\n    if len(all_files) != len(set(all_files)):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=(\"There are file name collisions; \"\n                                       \"consider using {_repindex}\"))\n        return\n\n    if dry_run:\n        for subpath in subpaths:\n            lgr.info(\"Would create a subdataset at %s\", subpath)\n        for row in rows:\n            lgr.info(\"Would download %s to %s\",\n                     row[\"url\"], os.path.join(dataset.path, row[\"filename\"]))\n            lgr.info(\"Metadata: %s\", row[\"meta_args\"])\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"ok\",\n                              message=\"dry-run finished\")\n        return\n\n    if not dataset.repo:\n        # Populate a new dataset with the URLs.\n        for r in dataset.create(result_xfm=None, return_type='generator'):\n            yield r\n\n    annex_options = [\"--fast\"] if fast else []\n\n    for spath in subpaths:\n        if os.path.exists(os.path.join(dataset.path, spath)):\n            lgr.warning(\n                \"Not creating subdataset at existing path: %s\",\n                spath)\n        else:\n            for r in dataset.create(spath, result_xfm=None,\n                                    return_type='generator'):\n                yield r\n\n    pbar_addurl = ui.get_progressbar(total=len(rows), label=\"Adding files\",\n                                     unit=\" Files\")\n    pbar_addurl.start()\n\n    files_to_add = []\n    meta_to_add = []\n    addurl_results = []\n    for row_idx, row in enumerate(rows, 1):\n        pbar_addurl.update(row_idx)\n        fname_abs = os.path.join(dataset.path, row[\"filename\"])\n        if row[\"subpath\"]:\n            # Adjust the dataset and filename for an `addurl` call\n            # from within the subdataset that will actually contain\n            # the link.\n            ds_current = Dataset(os.path.join(dataset.path, row[\"subpath\"]))\n            ds_filename = os.path.relpath(fname_abs, ds_current.path)\n        else:\n            ds_current = dataset\n            ds_filename = row[\"filename\"]\n\n        if os.path.exists(fname_abs) or os.path.islink(fname_abs):\n            if ifexists == \"skip\":\n                addurl_results.append(\n                    get_status_dict(action=\"addurls\",\n                                    ds=ds_current,\n                                    type=\"file\",\n                                    path=fname_abs,\n                                    status=\"notneeded\"))\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", fname_abs)\n                os.unlink(fname_abs)\n            else:\n                lgr.debug(\"File %s already exists\", fname_abs)\n\n        try:\n            ds_current.repo.add_url_to_file(ds_filename, row[\"url\"],\n                                            batch=True, options=annex_options)\n        except AnnexBatchCommandError:\n            addurl_results.append(\n                get_status_dict(action=\"addurls\",\n                                ds=ds_current,\n                                type=\"file\",\n                                path=os.path.join(ds_current.path,\n                                                  ds_filename),\n                                message=\"failed to add URL\",\n                                status=\"error\"))\n\n            continue\n        # Collect the status dicts to yield later so that we don't\n        # interrupt the progress bar.\n        addurl_results.append(\n            get_status_dict(action=\"addurls\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds_current.path,\n                                              ds_filename),\n                            status=\"ok\"))\n\n        files_to_add.append(row[\"filename\"])\n        meta_to_add.append((ds_current, ds_filename, row[\"meta_args\"]))\n    pbar_addurl.finish()\n\n    for result in addurl_results:\n        yield result\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n    if files_to_add:\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n    # TODO: Wrap this repeated \"delayed results/progress bar\" pattern.\n    pbar_meta = ui.get_progressbar(total=len(meta_to_add),\n                                   label=\"Adding metadata\", unit=\" Files\")\n    pbar_meta.start()\n\n    meta_results = []\n    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):\n        pbar_meta.update(meta_idx)\n        lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n        for arg in meta:\n            ds.repo._run_annex_command(\"metadata\",\n                                       annex_options=[\"--set\", arg, fname])\n        meta_results.append(\n            get_status_dict(action=\"addurls-metadata\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds.path, fname),\n                            message=\"added metadata\",\n                            status=\"ok\"))\n\n    for result in meta_results:\n        yield result\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "8f5ddebc8",
                "new_commit_hash": "a58191fae",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 289,
                            "length": 7
                        },
                        "new_range": {
                            "start": 289,
                            "length": 7
                        },
                        "section": "def get_url_names(url):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def extract(stream, input_type, filename_format, url_format,",
                                "type": "context"
                            },
                            {
                                "content": "            no_autometa, meta, missing_value):",
                                "type": "deleted"
                            },
                            {
                                "content": "            exclude_autometa, meta, missing_value):",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Extract and format information from `url_file`.",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    Parameters",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, filename_format, url_format,\n            no_autometa, meta, missing_value):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if not no_autometa:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        #\n        # TODO: Add a command line argument for excluding a particular\n        # column(s), maybe based on a regexp.\n        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())\n                                          if c != urlcol)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    infos = []\n    for row in rows:\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n        infos.append({\"url\": format_url(row),\n                      \"meta_args\": meta_args})\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    if any(i.startswith(\"_url\") for i in get_fmt_names(filename_format)):\n        for row, info in zip(rows, infos):\n            row.update(get_url_names(info[\"url\"]))\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows, infos)\n    return infos, subpaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 291,
                            "end_lineno": 346,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 311,
                            "length": 15
                        },
                        "new_range": {
                            "start": 311,
                            "length": 15
                        },
                        "section": "def extract(stream, input_type, filename_format, url_format,"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    format_url = partial(fmt.format, url_format)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    auto_meta_args = []",
                                "type": "context"
                            },
                            {
                                "content": "    if not no_autometa:",
                                "type": "deleted"
                            },
                            {
                                "content": "    if exclude_autometa not in [\"*\", \"\"]:",
                                "type": "added"
                            },
                            {
                                "content": "        urlcol = fmt_to_name(url_format, colidx_to_name)",
                                "type": "context"
                            },
                            {
                                "content": "        # TODO: Try to normalize invalid fields, checking for any",
                                "type": "context"
                            },
                            {
                                "content": "        # collisions.",
                                "type": "context"
                            },
                            {
                                "content": "        #",
                                "type": "deleted"
                            },
                            {
                                "content": "        # TODO: Add a command line argument for excluding a particular",
                                "type": "deleted"
                            },
                            {
                                "content": "        # column(s), maybe based on a regexp.",
                                "type": "deleted"
                            },
                            {
                                "content": "        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())",
                                "type": "deleted"
                            },
                            {
                                "content": "                                          if c != urlcol)",
                                "type": "deleted"
                            },
                            {
                                "content": "        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)",
                                "type": "added"
                            },
                            {
                                "content": "        if exclude_autometa:",
                                "type": "added"
                            },
                            {
                                "content": "            metacols = (c for c in metacols",
                                "type": "added"
                            },
                            {
                                "content": "                        if not re.search(exclude_autometa, c))",
                                "type": "added"
                            },
                            {
                                "content": "        metacols = filter_legal_metafield(metacols)",
                                "type": "added"
                            },
                            {
                                "content": "        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # Unlike `filename_format` and `url_format`, `meta` is a list",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, filename_format, url_format,\n            no_autometa, meta, missing_value):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if not no_autometa:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        #\n        # TODO: Add a command line argument for excluding a particular\n        # column(s), maybe based on a regexp.\n        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())\n                                          if c != urlcol)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    infos = []\n    for row in rows:\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n        infos.append({\"url\": format_url(row),\n                      \"meta_args\": meta_args})\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    if any(i.startswith(\"_url\") for i in get_fmt_names(filename_format)):\n        for row, info in zip(rows, infos):\n            row.update(get_url_names(info[\"url\"]))\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows, infos)\n    return infos, subpaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 291,
                            "end_lineno": 346,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 348,
                            "length": 7
                        },
                        "new_range": {
                            "start": 348,
                            "length": 7
                        },
                        "section": "def extract(stream, input_type, filename_format, url_format,"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",",
                                "type": "context"
                            },
                            {
                                "content": "             url_format=\"{0}\", filename_format=\"{1}\",",
                                "type": "context"
                            },
                            {
                                "content": "             no_autometa=False, meta=None,",
                                "type": "deleted"
                            },
                            {
                                "content": "             exclude_autometa=None, meta=None,",
                                "type": "added"
                            },
                            {
                                "content": "             message=None, dry_run=False, fast=False,",
                                "type": "context"
                            },
                            {
                                "content": "             ifexists=None, missing_value=None):",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"Create and update a dataset from a list of URLs.",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             no_autometa=False, meta=None,\n             message=None, dry_run=False, fast=False,\n             ifexists=None, missing_value=None):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file to\n        which the URL's content will be downloaded.  The file name may\n        contain directories.  The separator \"//\" can be used to indicate\n        that the left-side directory should be created as a new\n        subdataset.\n\n        In addition to the placeholders described in `url_format`, there\n        are a few special placeholder.\n\n          - \"_repindex\"\n\n            The constructed file names must be unique across all fields\n            rows.  To avoid collisions, the special placeholder\n            \"_repindex\" can added to the formatter.  It value will start\n            at 0 and increment every time a file name repeats.\n\n          - \"_urlN\"\n\n            Each part of the formatted URL is available.  For example,\n            in \"http://datalad.org/for/git-users\", \"_url0\" and \"_url1\"\n            would map to \"for\" and \"git-users\", respectively.\n\n            The final part is also available as \"_url_basename\".\n    no_autometa : bool, optional\n        By default, metadata field=value pairs are constructed with each\n        column in `url_file`, excluding any single column that is\n        specified via `url_format`.  Set this flag to True to disable\n        this behavior.  Metadata can still be set explicitly with the\n        `meta` argument.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should be\n        set the value of the fourth column.  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n    ifexists : {None, 'overwrite', 'skip'}\n        What to do if a constructed file name already exists.  The\n        default (None) behavior to proceed with the `git annex addurl`,\n        which will failed if the file size has changed.  If set to\n        'overwrite', remove the old file before adding the new one.  If\n        set to 'skip', do not add the new file.\n    missing_value : str, optional\n        When an empty string is encountered, use this value instead.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import AnnexBatchCommandError\n    from datalad.utils import assure_list\n    from datalad.ui import ui\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"not an annex repo\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        rows, subpaths = me.extract(fd, input_type,\n                                    filename_format, url_format,\n                                    no_autometa, meta,\n                                    missing_value)\n\n    all_files = [row[\"filename\"] for row in rows]\n    if len(all_files) != len(set(all_files)):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=(\"There are file name collisions; \"\n                                       \"consider using {_repindex}\"))\n        return\n\n    if dry_run:\n        for subpath in subpaths:\n            lgr.info(\"Would create a subdataset at %s\", subpath)\n        for row in rows:\n            lgr.info(\"Would download %s to %s\",\n                     row[\"url\"], os.path.join(dataset.path, row[\"filename\"]))\n            lgr.info(\"Metadata: %s\", row[\"meta_args\"])\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"ok\",\n                              message=\"dry-run finished\")\n        return\n\n    if not dataset.repo:\n        # Populate a new dataset with the URLs.\n        for r in dataset.create(result_xfm=None, return_type='generator'):\n            yield r\n\n    annex_options = [\"--fast\"] if fast else []\n\n    for spath in subpaths:\n        if os.path.exists(os.path.join(dataset.path, spath)):\n            lgr.warning(\n                \"Not creating subdataset at existing path: %s\",\n                spath)\n        else:\n            for r in dataset.create(spath, result_xfm=None,\n                                    return_type='generator'):\n                yield r\n\n    pbar_addurl = ui.get_progressbar(total=len(rows), label=\"Adding files\",\n                                     unit=\" Files\")\n    pbar_addurl.start()\n\n    files_to_add = []\n    meta_to_add = []\n    addurl_results = []\n    for row_idx, row in enumerate(rows, 1):\n        pbar_addurl.update(row_idx)\n        fname_abs = os.path.join(dataset.path, row[\"filename\"])\n        if row[\"subpath\"]:\n            # Adjust the dataset and filename for an `addurl` call\n            # from within the subdataset that will actually contain\n            # the link.\n            ds_current = Dataset(os.path.join(dataset.path, row[\"subpath\"]))\n            ds_filename = os.path.relpath(fname_abs, ds_current.path)\n        else:\n            ds_current = dataset\n            ds_filename = row[\"filename\"]\n\n        if os.path.exists(fname_abs) or os.path.islink(fname_abs):\n            if ifexists == \"skip\":\n                addurl_results.append(\n                    get_status_dict(action=\"addurls\",\n                                    ds=ds_current,\n                                    type=\"file\",\n                                    path=fname_abs,\n                                    status=\"notneeded\"))\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", fname_abs)\n                os.unlink(fname_abs)\n            else:\n                lgr.debug(\"File %s already exists\", fname_abs)\n\n        try:\n            ds_current.repo.add_url_to_file(ds_filename, row[\"url\"],\n                                            batch=True, options=annex_options)\n        except AnnexBatchCommandError:\n            addurl_results.append(\n                get_status_dict(action=\"addurls\",\n                                ds=ds_current,\n                                type=\"file\",\n                                path=os.path.join(ds_current.path,\n                                                  ds_filename),\n                                message=\"failed to add URL\",\n                                status=\"error\"))\n\n            continue\n        # Collect the status dicts to yield later so that we don't\n        # interrupt the progress bar.\n        addurl_results.append(\n            get_status_dict(action=\"addurls\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds_current.path,\n                                              ds_filename),\n                            status=\"ok\"))\n\n        files_to_add.append(row[\"filename\"])\n        meta_to_add.append((ds_current, ds_filename, row[\"meta_args\"]))\n    pbar_addurl.finish()\n\n    for result in addurl_results:\n        yield result\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n    if files_to_add:\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n    # TODO: Wrap this repeated \"delayed results/progress bar\" pattern.\n    pbar_meta = ui.get_progressbar(total=len(meta_to_add),\n                                   label=\"Adding metadata\", unit=\" Files\")\n    pbar_meta.start()\n\n    meta_results = []\n    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):\n        pbar_meta.update(meta_idx)\n        lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n        for arg in meta:\n            ds.repo._run_annex_command(\"metadata\",\n                                       annex_options=[\"--set\", arg, fname])\n        meta_results.append(\n            get_status_dict(action=\"addurls-metadata\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds.path, fname),\n                            message=\"added metadata\",\n                            status=\"ok\"))\n\n    for result in meta_results:\n        yield result",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 349,
                            "end_lineno": 647,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 403,
                            "length": 12
                        },
                        "new_range": {
                            "start": 403,
                            "length": 14
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "            would map to \"for\" and \"git-users\", respectively.",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "            The final part is also available as \"_url_basename\".",
                                "type": "context"
                            },
                            {
                                "content": "    no_autometa : bool, optional",
                                "type": "deleted"
                            },
                            {
                                "content": "    exclude_autometa : str, optional",
                                "type": "added"
                            },
                            {
                                "content": "        By default, metadata field=value pairs are constructed with each",
                                "type": "context"
                            },
                            {
                                "content": "        column in `url_file`, excluding any single column that is",
                                "type": "context"
                            },
                            {
                                "content": "        specified via `url_format`.  Set this flag to True to disable",
                                "type": "deleted"
                            },
                            {
                                "content": "        this behavior.  Metadata can still be set explicitly with the",
                                "type": "deleted"
                            },
                            {
                                "content": "        `meta` argument.",
                                "type": "deleted"
                            },
                            {
                                "content": "        specified via `url_format`.  This argument can be used to",
                                "type": "added"
                            },
                            {
                                "content": "        exclude columns that match a regular expression.  If set to '*'",
                                "type": "added"
                            },
                            {
                                "content": "        or an empty string, automatic metadata extraction is disabled",
                                "type": "added"
                            },
                            {
                                "content": "        completely.  This argument does not affect metadata set",
                                "type": "added"
                            },
                            {
                                "content": "        explicitly with the `meta` argument.",
                                "type": "added"
                            },
                            {
                                "content": "    meta : str, optional",
                                "type": "context"
                            },
                            {
                                "content": "        A format string that specifies metadata.  It should be",
                                "type": "context"
                            },
                            {
                                "content": "        structured as \"<field>=<value>\".  The same placeholders from",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             no_autometa=False, meta=None,\n             message=None, dry_run=False, fast=False,\n             ifexists=None, missing_value=None):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file to\n        which the URL's content will be downloaded.  The file name may\n        contain directories.  The separator \"//\" can be used to indicate\n        that the left-side directory should be created as a new\n        subdataset.\n\n        In addition to the placeholders described in `url_format`, there\n        are a few special placeholder.\n\n          - \"_repindex\"\n\n            The constructed file names must be unique across all fields\n            rows.  To avoid collisions, the special placeholder\n            \"_repindex\" can added to the formatter.  It value will start\n            at 0 and increment every time a file name repeats.\n\n          - \"_urlN\"\n\n            Each part of the formatted URL is available.  For example,\n            in \"http://datalad.org/for/git-users\", \"_url0\" and \"_url1\"\n            would map to \"for\" and \"git-users\", respectively.\n\n            The final part is also available as \"_url_basename\".\n    no_autometa : bool, optional\n        By default, metadata field=value pairs are constructed with each\n        column in `url_file`, excluding any single column that is\n        specified via `url_format`.  Set this flag to True to disable\n        this behavior.  Metadata can still be set explicitly with the\n        `meta` argument.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should be\n        set the value of the fourth column.  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n    ifexists : {None, 'overwrite', 'skip'}\n        What to do if a constructed file name already exists.  The\n        default (None) behavior to proceed with the `git annex addurl`,\n        which will failed if the file size has changed.  If set to\n        'overwrite', remove the old file before adding the new one.  If\n        set to 'skip', do not add the new file.\n    missing_value : str, optional\n        When an empty string is encountered, use this value instead.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import AnnexBatchCommandError\n    from datalad.utils import assure_list\n    from datalad.ui import ui\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"not an annex repo\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        rows, subpaths = me.extract(fd, input_type,\n                                    filename_format, url_format,\n                                    no_autometa, meta,\n                                    missing_value)\n\n    all_files = [row[\"filename\"] for row in rows]\n    if len(all_files) != len(set(all_files)):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=(\"There are file name collisions; \"\n                                       \"consider using {_repindex}\"))\n        return\n\n    if dry_run:\n        for subpath in subpaths:\n            lgr.info(\"Would create a subdataset at %s\", subpath)\n        for row in rows:\n            lgr.info(\"Would download %s to %s\",\n                     row[\"url\"], os.path.join(dataset.path, row[\"filename\"]))\n            lgr.info(\"Metadata: %s\", row[\"meta_args\"])\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"ok\",\n                              message=\"dry-run finished\")\n        return\n\n    if not dataset.repo:\n        # Populate a new dataset with the URLs.\n        for r in dataset.create(result_xfm=None, return_type='generator'):\n            yield r\n\n    annex_options = [\"--fast\"] if fast else []\n\n    for spath in subpaths:\n        if os.path.exists(os.path.join(dataset.path, spath)):\n            lgr.warning(\n                \"Not creating subdataset at existing path: %s\",\n                spath)\n        else:\n            for r in dataset.create(spath, result_xfm=None,\n                                    return_type='generator'):\n                yield r\n\n    pbar_addurl = ui.get_progressbar(total=len(rows), label=\"Adding files\",\n                                     unit=\" Files\")\n    pbar_addurl.start()\n\n    files_to_add = []\n    meta_to_add = []\n    addurl_results = []\n    for row_idx, row in enumerate(rows, 1):\n        pbar_addurl.update(row_idx)\n        fname_abs = os.path.join(dataset.path, row[\"filename\"])\n        if row[\"subpath\"]:\n            # Adjust the dataset and filename for an `addurl` call\n            # from within the subdataset that will actually contain\n            # the link.\n            ds_current = Dataset(os.path.join(dataset.path, row[\"subpath\"]))\n            ds_filename = os.path.relpath(fname_abs, ds_current.path)\n        else:\n            ds_current = dataset\n            ds_filename = row[\"filename\"]\n\n        if os.path.exists(fname_abs) or os.path.islink(fname_abs):\n            if ifexists == \"skip\":\n                addurl_results.append(\n                    get_status_dict(action=\"addurls\",\n                                    ds=ds_current,\n                                    type=\"file\",\n                                    path=fname_abs,\n                                    status=\"notneeded\"))\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", fname_abs)\n                os.unlink(fname_abs)\n            else:\n                lgr.debug(\"File %s already exists\", fname_abs)\n\n        try:\n            ds_current.repo.add_url_to_file(ds_filename, row[\"url\"],\n                                            batch=True, options=annex_options)\n        except AnnexBatchCommandError:\n            addurl_results.append(\n                get_status_dict(action=\"addurls\",\n                                ds=ds_current,\n                                type=\"file\",\n                                path=os.path.join(ds_current.path,\n                                                  ds_filename),\n                                message=\"failed to add URL\",\n                                status=\"error\"))\n\n            continue\n        # Collect the status dicts to yield later so that we don't\n        # interrupt the progress bar.\n        addurl_results.append(\n            get_status_dict(action=\"addurls\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds_current.path,\n                                              ds_filename),\n                            status=\"ok\"))\n\n        files_to_add.append(row[\"filename\"])\n        meta_to_add.append((ds_current, ds_filename, row[\"meta_args\"]))\n    pbar_addurl.finish()\n\n    for result in addurl_results:\n        yield result\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n    if files_to_add:\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n    # TODO: Wrap this repeated \"delayed results/progress bar\" pattern.\n    pbar_meta = ui.get_progressbar(total=len(meta_to_add),\n                                   label=\"Adding metadata\", unit=\" Files\")\n    pbar_meta.start()\n\n    meta_results = []\n    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):\n        pbar_meta.update(meta_idx)\n        lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n        for arg in meta:\n            ds.repo._run_annex_command(\"metadata\",\n                                       annex_options=[\"--set\", arg, fname])\n        meta_results.append(\n            get_status_dict(action=\"addurls-metadata\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds.path, fname),\n                            message=\"added metadata\",\n                            status=\"ok\"))\n\n    for result in meta_results:\n        yield result",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 349,
                            "end_lineno": 647,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 504,
                            "length": 7
                        },
                        "new_range": {
                            "start": 506,
                            "length": 7
                        },
                        "section": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    with open(url_file) as fd:",
                                "type": "context"
                            },
                            {
                                "content": "        rows, subpaths = me.extract(fd, input_type,",
                                "type": "context"
                            },
                            {
                                "content": "                                    filename_format, url_format,",
                                "type": "context"
                            },
                            {
                                "content": "                                    no_autometa, meta,",
                                "type": "deleted"
                            },
                            {
                                "content": "                                    exclude_autometa, meta,",
                                "type": "added"
                            },
                            {
                                "content": "                                    missing_value)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    all_files = [row[\"filename\"] for row in rows]",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "dlplugin",
                            "content": "def dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             no_autometa=False, meta=None,\n             message=None, dry_run=False, fast=False,\n             ifexists=None, missing_value=None):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file to\n        which the URL's content will be downloaded.  The file name may\n        contain directories.  The separator \"//\" can be used to indicate\n        that the left-side directory should be created as a new\n        subdataset.\n\n        In addition to the placeholders described in `url_format`, there\n        are a few special placeholder.\n\n          - \"_repindex\"\n\n            The constructed file names must be unique across all fields\n            rows.  To avoid collisions, the special placeholder\n            \"_repindex\" can added to the formatter.  It value will start\n            at 0 and increment every time a file name repeats.\n\n          - \"_urlN\"\n\n            Each part of the formatted URL is available.  For example,\n            in \"http://datalad.org/for/git-users\", \"_url0\" and \"_url1\"\n            would map to \"for\" and \"git-users\", respectively.\n\n            The final part is also available as \"_url_basename\".\n    no_autometa : bool, optional\n        By default, metadata field=value pairs are constructed with each\n        column in `url_file`, excluding any single column that is\n        specified via `url_format`.  Set this flag to True to disable\n        this behavior.  Metadata can still be set explicitly with the\n        `meta` argument.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should be\n        set the value of the fourth column.  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n    ifexists : {None, 'overwrite', 'skip'}\n        What to do if a constructed file name already exists.  The\n        default (None) behavior to proceed with the `git annex addurl`,\n        which will failed if the file size has changed.  If set to\n        'overwrite', remove the old file before adding the new one.  If\n        set to 'skip', do not add the new file.\n    missing_value : str, optional\n        When an empty string is encountered, use this value instead.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import AnnexBatchCommandError\n    from datalad.utils import assure_list\n    from datalad.ui import ui\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"not an annex repo\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        rows, subpaths = me.extract(fd, input_type,\n                                    filename_format, url_format,\n                                    no_autometa, meta,\n                                    missing_value)\n\n    all_files = [row[\"filename\"] for row in rows]\n    if len(all_files) != len(set(all_files)):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=(\"There are file name collisions; \"\n                                       \"consider using {_repindex}\"))\n        return\n\n    if dry_run:\n        for subpath in subpaths:\n            lgr.info(\"Would create a subdataset at %s\", subpath)\n        for row in rows:\n            lgr.info(\"Would download %s to %s\",\n                     row[\"url\"], os.path.join(dataset.path, row[\"filename\"]))\n            lgr.info(\"Metadata: %s\", row[\"meta_args\"])\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"ok\",\n                              message=\"dry-run finished\")\n        return\n\n    if not dataset.repo:\n        # Populate a new dataset with the URLs.\n        for r in dataset.create(result_xfm=None, return_type='generator'):\n            yield r\n\n    annex_options = [\"--fast\"] if fast else []\n\n    for spath in subpaths:\n        if os.path.exists(os.path.join(dataset.path, spath)):\n            lgr.warning(\n                \"Not creating subdataset at existing path: %s\",\n                spath)\n        else:\n            for r in dataset.create(spath, result_xfm=None,\n                                    return_type='generator'):\n                yield r\n\n    pbar_addurl = ui.get_progressbar(total=len(rows), label=\"Adding files\",\n                                     unit=\" Files\")\n    pbar_addurl.start()\n\n    files_to_add = []\n    meta_to_add = []\n    addurl_results = []\n    for row_idx, row in enumerate(rows, 1):\n        pbar_addurl.update(row_idx)\n        fname_abs = os.path.join(dataset.path, row[\"filename\"])\n        if row[\"subpath\"]:\n            # Adjust the dataset and filename for an `addurl` call\n            # from within the subdataset that will actually contain\n            # the link.\n            ds_current = Dataset(os.path.join(dataset.path, row[\"subpath\"]))\n            ds_filename = os.path.relpath(fname_abs, ds_current.path)\n        else:\n            ds_current = dataset\n            ds_filename = row[\"filename\"]\n\n        if os.path.exists(fname_abs) or os.path.islink(fname_abs):\n            if ifexists == \"skip\":\n                addurl_results.append(\n                    get_status_dict(action=\"addurls\",\n                                    ds=ds_current,\n                                    type=\"file\",\n                                    path=fname_abs,\n                                    status=\"notneeded\"))\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", fname_abs)\n                os.unlink(fname_abs)\n            else:\n                lgr.debug(\"File %s already exists\", fname_abs)\n\n        try:\n            ds_current.repo.add_url_to_file(ds_filename, row[\"url\"],\n                                            batch=True, options=annex_options)\n        except AnnexBatchCommandError:\n            addurl_results.append(\n                get_status_dict(action=\"addurls\",\n                                ds=ds_current,\n                                type=\"file\",\n                                path=os.path.join(ds_current.path,\n                                                  ds_filename),\n                                message=\"failed to add URL\",\n                                status=\"error\"))\n\n            continue\n        # Collect the status dicts to yield later so that we don't\n        # interrupt the progress bar.\n        addurl_results.append(\n            get_status_dict(action=\"addurls\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds_current.path,\n                                              ds_filename),\n                            status=\"ok\"))\n\n        files_to_add.append(row[\"filename\"])\n        meta_to_add.append((ds_current, ds_filename, row[\"meta_args\"]))\n    pbar_addurl.finish()\n\n    for result in addurl_results:\n        yield result\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n    if files_to_add:\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n    # TODO: Wrap this repeated \"delayed results/progress bar\" pattern.\n    pbar_meta = ui.get_progressbar(total=len(meta_to_add),\n                                   label=\"Adding metadata\", unit=\" Files\")\n    pbar_meta.start()\n\n    meta_results = []\n    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):\n        pbar_meta.update(meta_idx)\n        lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n        for arg in meta:\n            ds.repo._run_annex_command(\"metadata\",\n                                       annex_options=[\"--set\", arg, fname])\n        meta_results.append(\n            get_status_dict(action=\"addurls-metadata\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds.path, fname),\n                            message=\"added metadata\",\n                            status=\"ok\"))\n\n    for result in meta_results:\n        yield result",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 349,
                            "end_lineno": 647,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport tempfile\n\nfrom six.moves import StringIO\n\nfrom datalad.api import Dataset, plugin, subdatasets\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.tests.utils import chpwd, slow, swallow_logs\nfrom datalad.tests.utils import assert_false, assert_true, assert_raises\nfrom datalad.tests.utils import assert_in, assert_in_results, assert_dict_equal\nfrom datalad.tests.utils import eq_, ok_exists\nfrom datalad.tests.utils import create_tree, with_tree, with_tempfile, HTTPPath\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\n\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = addurls.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = addurls.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = addurls.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        eq_(list(addurls.clean_meta_args(args)), expect)\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"noequal\"]))\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\ndef test_get_subpaths():\n    for fname, expect in [(\"no/dbl/slash\", (\"no/dbl/slash\", [])),\n                          (\"p1//n\", (\"p1/n\", [\"p1\"])),\n                          (\"p1//p2/p3//n\", (\"p1/p2/p3/n\",\n                                            [\"p1\", \"p1/p2/p3\"])),\n                          (\"//n\", (\"/n\", [\"\"])),\n                          (\"n//\", (\"n/\", [\"n\"]))]:\n        eq_(addurls.get_subpaths(fname), expect)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(addurls.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(addurls.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(addurls.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(addurls.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(addurls.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(addurls.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(addurls.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(addurls.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(addurls.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(addurls.fmt_to_name(\"\", {}))\n    assert_false(addurls.fmt_to_name(\"nonames\", {}))\n    assert_false(addurls.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_url_names():\n    eq_(addurls.get_url_names(\"\"), {})\n    eq_(addurls.get_url_names(\"http://datalad.org\"), {})\n\n    assert_dict_equal(addurls.get_url_names(\"http://datalad.org/about.html\"),\n                      {\"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\"})\n    assert_dict_equal(addurls.get_url_names(\"http://datalad.org/about.html\"),\n                      addurls.get_url_names(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        addurls.get_url_names(\"http://datalad.org/for/git-users\"),\n        {\"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        False, [], None)\n\n    eq_(subpaths,\n        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    eq_([set(d[\"meta_args\"]) for d in info],\n        [{\"name=will\", \"age_group=kid\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=bob\", \"age_group=adult\", \"debut_season=2\", \"now_dead=yes\"},\n         {\"name=scott\", \"age_group=adult\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=max\", \"age_group=kid\", \"debut_season=2\", \"now_dead=no\"}])\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])\n\n\ndef test_extract_no_autometa():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        True,\n        [\"group={age_group}\"],\n        None)\n\n    eq_([d[\"meta_args\"] for d in info],\n        [[\"group=kid\"], [\"group=adult\"], [\"group=adult\"], [\"group=kid\"]])\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            False,\n            [\"group={age_group}\"],\n            None]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  addurls.extract,\n                  None, \"not_csv_or_json\", None, None, None, None, None)\n\n\ndef test_addurls_no_urlfile():\n    with assert_raises(IncompleteResultsError) as raised:\n        plugin(\"addurls\")\n    assert_in(\"Must specify url_file argument\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, no_annex=True)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.plugin(\"addurls\", url_file=\"dummy\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    with chpwd(path):\n        json_file = \"links.json\"\n        with open(json_file, \"w\") as jfh:\n            json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                       {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                       {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                      jfh)\n\n        ds.add(\".\", message=\"setup\")\n\n        with swallow_logs(new_level=logging.INFO) as cml:\n            ds.plugin(\"addurls\", url_file=json_file, url_format=\"{url}\",\n                      filename_format=\"{subdir}//{name}\", dry_run=True)\n\n            for dir_ in [\"foo\", \"bar\"]:\n                assert_in(\"Would create a subdataset at {}\".format(dir_),\n                          cml.out)\n            assert_in(\n                \"Would download URL/a.dat to {}\".format(\n                    os.path.join(path, \"foo\", \"a\")),\n                cml.out)\n\n            assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                      cml.out)\n\n\n@slow  # ~9s\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\": x + \" content\" for x in \"abcd\"}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            meta_results = dict(\n                zip(filenames,\n                    ds.repo._run_annex_command_json(\"metadata\",\n                                                    files=filenames)))\n\n            for fname, subdir in zip(filenames, [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(\n                    {k: v for k, v in meta_results[fname][\"fields\"].items()\n                     if not k.endswith(\"lastchanged\")},\n                    {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\",\n                          ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\",\n                          ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.plugin,\n                          \"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        plugin(\"addurls\",\n               dataset=dspath,\n               url_file=self.json_file,\n               url_format=\"{url}\", filename_format=\"{name}\")\n\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{subdir}//{name}\")\n\n            for fname in [\"foo/a\", \"bar/b\", \"foo/c\"]:\n                ok_exists(fname)\n\n            eq_(set(subdatasets(ds, recursive=True, result_xfm=\"relpaths\")),\n                {\"foo\", \"bar\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\",\n                          filename_format=\"{subdir}//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\",\n                      filename_format=\"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\",\n                      filename_format=\"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n",
            "new_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport tempfile\n\nfrom six.moves import StringIO\n\nfrom datalad.api import Dataset, plugin, subdatasets\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.tests.utils import chpwd, slow, swallow_logs\nfrom datalad.tests.utils import assert_false, assert_true, assert_raises\nfrom datalad.tests.utils import assert_in, assert_in_results, assert_dict_equal\nfrom datalad.tests.utils import eq_, ok_exists\nfrom datalad.tests.utils import create_tree, with_tree, with_tempfile, HTTPPath\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\n\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = addurls.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = addurls.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = addurls.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        eq_(list(addurls.clean_meta_args(args)), expect)\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"noequal\"]))\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\ndef test_get_subpaths():\n    for fname, expect in [(\"no/dbl/slash\", (\"no/dbl/slash\", [])),\n                          (\"p1//n\", (\"p1/n\", [\"p1\"])),\n                          (\"p1//p2/p3//n\", (\"p1/p2/p3/n\",\n                                            [\"p1\", \"p1/p2/p3\"])),\n                          (\"//n\", (\"/n\", [\"\"])),\n                          (\"n//\", (\"n/\", [\"n\"]))]:\n        eq_(addurls.get_subpaths(fname), expect)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(addurls.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(addurls.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(addurls.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(addurls.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(addurls.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(addurls.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(addurls.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(addurls.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(addurls.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(addurls.fmt_to_name(\"\", {}))\n    assert_false(addurls.fmt_to_name(\"nonames\", {}))\n    assert_false(addurls.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_url_names():\n    eq_(addurls.get_url_names(\"\"), {})\n    eq_(addurls.get_url_names(\"http://datalad.org\"), {})\n\n    assert_dict_equal(addurls.get_url_names(\"http://datalad.org/about.html\"),\n                      {\"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\"})\n    assert_dict_equal(addurls.get_url_names(\"http://datalad.org/about.html\"),\n                      addurls.get_url_names(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        addurls.get_url_names(\"http://datalad.org/for/git-users\"),\n        {\"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        None, [], None)\n\n    eq_(subpaths,\n        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    eq_([set(d[\"meta_args\"]) for d in info],\n        [{\"name=will\", \"age_group=kid\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=bob\", \"age_group=adult\", \"debut_season=2\", \"now_dead=yes\"},\n         {\"name=scott\", \"age_group=adult\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=max\", \"age_group=kid\", \"debut_season=2\", \"now_dead=no\"}])\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])\n\n\ndef test_extract_disable_autometa():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        \"*\",\n        [\"group={age_group}\"],\n        None)\n\n    eq_([d[\"meta_args\"] for d in info],\n        [[\"group=kid\"], [\"group=adult\"], [\"group=adult\"], [\"group=kid\"]])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        \"ea\",\n        [],\n        None)\n\n    eq_([set(d[\"meta_args\"]) for d in info],\n        [{\"name=will\", \"age_group=kid\"},\n         {\"name=bob\", \"age_group=adult\"},\n         {\"name=scott\", \"age_group=adult\"},\n         {\"name=max\", \"age_group=kid\"}])\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            None,\n            [\"group={age_group}\"],\n            None]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  addurls.extract,\n                  None, \"not_csv_or_json\", None, None, None, None, None)\n\n\ndef test_addurls_no_urlfile():\n    with assert_raises(IncompleteResultsError) as raised:\n        plugin(\"addurls\")\n    assert_in(\"Must specify url_file argument\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, no_annex=True)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.plugin(\"addurls\", url_file=\"dummy\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    with chpwd(path):\n        json_file = \"links.json\"\n        with open(json_file, \"w\") as jfh:\n            json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                       {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                       {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                      jfh)\n\n        ds.add(\".\", message=\"setup\")\n\n        with swallow_logs(new_level=logging.INFO) as cml:\n            ds.plugin(\"addurls\", url_file=json_file, url_format=\"{url}\",\n                      filename_format=\"{subdir}//{name}\", dry_run=True)\n\n            for dir_ in [\"foo\", \"bar\"]:\n                assert_in(\"Would create a subdataset at {}\".format(dir_),\n                          cml.out)\n            assert_in(\n                \"Would download URL/a.dat to {}\".format(\n                    os.path.join(path, \"foo\", \"a\")),\n                cml.out)\n\n            assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                      cml.out)\n\n\n@slow  # ~9s\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\": x + \" content\" for x in \"abcd\"}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            meta_results = dict(\n                zip(filenames,\n                    ds.repo._run_annex_command_json(\"metadata\",\n                                                    files=filenames)))\n\n            for fname, subdir in zip(filenames, [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(\n                    {k: v for k, v in meta_results[fname][\"fields\"].items()\n                     if not k.endswith(\"lastchanged\")},\n                    {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\",\n                          ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\",\n                          ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.plugin,\n                          \"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        plugin(\"addurls\",\n               dataset=dspath,\n               url_file=self.json_file,\n               url_format=\"{url}\", filename_format=\"{name}\")\n\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{subdir}//{name}\")\n\n            for fname in [\"foo/a\", \"bar/b\", \"foo/c\"]:\n                ok_exists(fname)\n\n            eq_(set(subdatasets(ds, recursive=True, result_xfm=\"relpaths\")),\n                {\"foo\", \"bar\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\",\n                          filename_format=\"{subdir}//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\",\n                      filename_format=\"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\",\n                      filename_format=\"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/tests/test_addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "f523d964f",
                "new_commit_hash": "ef4030178",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/tests/test_addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/tests/test_addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 189,
                            "length": 7
                        },
                        "new_range": {
                            "start": 189,
                            "length": 7
                        },
                        "section": "def test_extract():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "        False, [], None)",
                                "type": "deleted"
                            },
                            {
                                "content": "        None, [], None)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(subpaths,",
                                "type": "context"
                            },
                            {
                                "content": "        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract",
                            "content": "def test_extract():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        False, [], None)\n\n    eq_(subpaths,\n        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    eq_([set(d[\"meta_args\"]) for d in info],\n        [{\"name=will\", \"age_group=kid\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=bob\", \"age_group=adult\", \"debut_season=2\", \"now_dead=yes\"},\n         {\"name=scott\", \"age_group=adult\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=max\", \"age_group=kid\", \"debut_season=2\", \"now_dead=no\"}])\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 187,
                            "end_lineno": 211,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 211,
                            "length": 12
                        },
                        "new_range": {
                            "start": 211,
                            "length": 12
                        },
                        "section": "def test_extract():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract_no_autometa():",
                                "type": "deleted"
                            },
                            {
                                "content": "def test_extract_disable_autometa():",
                                "type": "added"
                            },
                            {
                                "content": "    info, subpaths = addurls.extract(",
                                "type": "context"
                            },
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "        \"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "        True,",
                                "type": "deleted"
                            },
                            {
                                "content": "        \"*\",",
                                "type": "added"
                            },
                            {
                                "content": "        [\"group={age_group}\"],",
                                "type": "context"
                            },
                            {
                                "content": "        None)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_disable_autometa",
                            "content": "def test_extract_disable_autometa():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        \"*\",\n        [\"group={age_group}\"],\n        None)\n\n    eq_([d[\"meta_args\"] for d in info],\n        [[\"group=kid\"], [\"group=adult\"], [\"group=adult\"], [\"group=kid\"]])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 214,
                            "end_lineno": 224,
                            "parent": null
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_no_autometa",
                            "content": "def test_extract_no_autometa():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        True,\n        [\"group={age_group}\"],\n        None)\n\n    eq_([d[\"meta_args\"] for d in info],\n        [[\"group=kid\"], [\"group=adult\"], [\"group=adult\"], [\"group=kid\"]])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 214,
                            "end_lineno": 224,
                            "parent": null
                        }
                    ]
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 224,
                            "length": 6
                        },
                        "new_range": {
                            "start": 224,
                            "length": 21
                        },
                        "section": "def test_extract_no_autometa():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        [[\"group=kid\"], [\"group=adult\"], [\"group=adult\"], [\"group=kid\"]])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract_exclude_autometa_regexp():",
                                "type": "added"
                            },
                            {
                                "content": "    info, subpaths = addurls.extract(",
                                "type": "added"
                            },
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "added"
                            },
                            {
                                "content": "        \"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "added"
                            },
                            {
                                "content": "        \"{name}_{debut_season}.com\",",
                                "type": "added"
                            },
                            {
                                "content": "        \"ea\",",
                                "type": "added"
                            },
                            {
                                "content": "        [],",
                                "type": "added"
                            },
                            {
                                "content": "        None)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([set(d[\"meta_args\"]) for d in info],",
                                "type": "added"
                            },
                            {
                                "content": "        [{\"name=will\", \"age_group=kid\"},",
                                "type": "added"
                            },
                            {
                                "content": "         {\"name=bob\", \"age_group=adult\"},",
                                "type": "added"
                            },
                            {
                                "content": "         {\"name=scott\", \"age_group=adult\"},",
                                "type": "added"
                            },
                            {
                                "content": "         {\"name=max\", \"age_group=kid\"}])",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def test_extract_csv_json_equal():",
                                "type": "context"
                            },
                            {
                                "content": "    keys = ST_DATA[\"header\"]",
                                "type": "context"
                            },
                            {
                                "content": "    csv_rows = [\",\".join(keys)]",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_exclude_autometa_regexp",
                            "content": "def test_extract_exclude_autometa_regexp():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        \"ea\",\n        [],\n        None)\n\n    eq_([set(d[\"meta_args\"]) for d in info],\n        [{\"name=will\", \"age_group=kid\"},\n         {\"name=bob\", \"age_group=adult\"},\n         {\"name=scott\", \"age_group=adult\"},\n         {\"name=max\", \"age_group=kid\"}])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 227,
                            "end_lineno": 240,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 232,
                            "length": 7
                        },
                        "new_range": {
                            "start": 247,
                            "length": 7
                        },
                        "section": "def test_extract_csv_json_equal():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    args = [\"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "            \"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "            False,",
                                "type": "deleted"
                            },
                            {
                                "content": "            None,",
                                "type": "added"
                            },
                            {
                                "content": "            [\"group={age_group}\"],",
                                "type": "context"
                            },
                            {
                                "content": "            None]",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_csv_json_equal",
                            "content": "def test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            False,\n            [\"group={age_group}\"],\n            None]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    eq_(json_output, csv_output)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 227,
                            "end_lineno": 242,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "4476540ed403fd0dea8e1d717ff85ae1d56ae6f3^",
    "new_commit_hash": "4476540ed403fd0dea8e1d717ff85ae1d56ae6f3",
    "commit_message": "ENH: addurls: Support excluding auto-metadata columns by regexp",
    "commit_date": "2018-02-27T09:07:21-05:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "4476540ed403fd0dea8e1d717ff85ae1d56ae6f3",
    "test_file_codes": [
        "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport tempfile\n\nfrom six.moves import StringIO\n\nfrom datalad.api import Dataset, plugin, subdatasets\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.tests.utils import chpwd, slow, swallow_logs\nfrom datalad.tests.utils import assert_false, assert_true, assert_raises\nfrom datalad.tests.utils import assert_in, assert_in_results, assert_dict_equal\nfrom datalad.tests.utils import eq_, ok_exists\nfrom datalad.tests.utils import create_tree, with_tree, with_tempfile, HTTPPath\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\n\nfrom datalad.plugin import addurls\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = addurls.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = addurls.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = addurls.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = addurls.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = addurls.Formatter({})\n    fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}) == \"value0\"\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = addurls.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\") == \"x\"\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\") == \"x    \"\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = addurls.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = addurls.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], []),\n                         ([\" field=yes \"], [\"field=yes\"]),\n                         ([\"field= value=\"], [\"field=value=\"])]:\n        eq_(list(addurls.clean_meta_args(args)), expect)\n\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"noequal\"]))\n    assert_raises(ValueError,\n                  list,\n                  addurls.clean_meta_args([\"=value\"]))\n\n\ndef test_get_subpaths():\n    for fname, expect in [(\"no/dbl/slash\", (\"no/dbl/slash\", [])),\n                          (\"p1//n\", (\"p1/n\", [\"p1\"])),\n                          (\"p1//p2/p3//n\", (\"p1/p2/p3/n\",\n                                            [\"p1\", \"p1/p2/p3\"])),\n                          (\"//n\", (\"/n\", [\"\"])),\n                          (\"n//\", (\"n/\", [\"n\"]))]:\n        eq_(addurls.get_subpaths(fname), expect)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(addurls.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(addurls.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(addurls.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(addurls.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(addurls.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(addurls.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(addurls.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(addurls.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(addurls.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(addurls.fmt_to_name(\"\", {}))\n    assert_false(addurls.fmt_to_name(\"nonames\", {}))\n    assert_false(addurls.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_url_names():\n    eq_(addurls.get_url_names(\"\"), {})\n    eq_(addurls.get_url_names(\"http://datalad.org\"), {})\n\n    assert_dict_equal(addurls.get_url_names(\"http://datalad.org/about.html\"),\n                      {\"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\"})\n    assert_dict_equal(addurls.get_url_names(\"http://datalad.org/about.html\"),\n                      addurls.get_url_names(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        addurls.get_url_names(\"http://datalad.org/for/git-users\"),\n        {\"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        None, [], None)\n\n    eq_(subpaths,\n        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    eq_([set(d[\"meta_args\"]) for d in info],\n        [{\"name=will\", \"age_group=kid\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=bob\", \"age_group=adult\", \"debut_season=2\", \"now_dead=yes\"},\n         {\"name=scott\", \"age_group=adult\", \"debut_season=1\", \"now_dead=no\"},\n         {\"name=max\", \"age_group=kid\", \"debut_season=2\", \"now_dead=no\"}])\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])\n\n\ndef test_extract_disable_autometa():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        \"*\",\n        [\"group={age_group}\"],\n        None)\n\n    eq_([d[\"meta_args\"] for d in info],\n        [[\"group=kid\"], [\"group=adult\"], [\"group=adult\"], [\"group=kid\"]])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, subpaths = addurls.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        \"{age_group}//{now_dead}//{name}.csv\",\n        \"{name}_{debut_season}.com\",\n        \"ea\",\n        [],\n        None)\n\n    eq_([set(d[\"meta_args\"]) for d in info],\n        [{\"name=will\", \"age_group=kid\"},\n         {\"name=bob\", \"age_group=adult\"},\n         {\"name=scott\", \"age_group=adult\"},\n         {\"name=max\", \"age_group=kid\"}])\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    args = [\"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            None,\n            [\"group={age_group}\"],\n            None]\n\n    json_output = addurls.extract(json_stream(ST_DATA[\"rows\"]), \"json\", *args)\n    csv_output = addurls.extract(csv_rows, \"csv\", *args)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  addurls.extract,\n                  None, \"not_csv_or_json\", None, None, None, None, None)\n\n\ndef test_addurls_no_urlfile():\n    with assert_raises(IncompleteResultsError) as raised:\n        plugin(\"addurls\")\n    assert_in(\"Must specify url_file argument\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, no_annex=True)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.plugin(\"addurls\", url_file=\"dummy\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    with chpwd(path):\n        json_file = \"links.json\"\n        with open(json_file, \"w\") as jfh:\n            json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                       {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                       {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                      jfh)\n\n        ds.add(\".\", message=\"setup\")\n\n        with swallow_logs(new_level=logging.INFO) as cml:\n            ds.plugin(\"addurls\", url_file=json_file, url_format=\"{url}\",\n                      filename_format=\"{subdir}//{name}\", dry_run=True)\n\n            for dir_ in [\"foo\", \"bar\"]:\n                assert_in(\"Would create a subdataset at {}\".format(dir_),\n                          cml.out)\n            assert_in(\n                \"Would download URL/a.dat to {}\".format(\n                    os.path.join(path, \"foo\", \"a\")),\n                cml.out)\n\n            assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                      cml.out)\n\n\n@slow  # ~9s\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\": x + \" content\" for x in \"abcd\"}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            meta_results = dict(\n                zip(filenames,\n                    ds.repo._run_annex_command_json(\"metadata\",\n                                                    files=filenames)))\n\n            for fname, subdir in zip(filenames, [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(\n                    {k: v for k, v in meta_results[fname][\"fields\"].items()\n                     if not k.endswith(\"lastchanged\")},\n                    {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\",\n                          ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\",\n                          ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.plugin,\n                          \"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        plugin(\"addurls\",\n               dataset=dspath,\n               url_file=self.json_file,\n               url_format=\"{url}\", filename_format=\"{name}\")\n\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\", filename_format=\"{subdir}//{name}\")\n\n            for fname in [\"foo/a\", \"bar/b\", \"foo/c\"]:\n                ok_exists(fname)\n\n            eq_(set(subdatasets(ds, recursive=True, result_xfm=\"relpaths\")),\n                {\"foo\", \"bar\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\",\n                          filename_format=\"{subdir}//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.plugin(\"addurls\", url_file=self.json_file,\n                          url_format=\"{url}\", filename_format=\"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\",\n                      filename_format=\"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.plugin(\"addurls\", url_file=self.json_file,\n                      url_format=\"{url}\",\n                      filename_format=\"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.9.1.dev931\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 133ms\nInstalled 25 packages in 4.70s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp13Efii/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp13Efii/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp13Efii/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp13Efii/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp13Efii/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpca84FU/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpca84FU/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpca84FU/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpca84FU/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpca84FU/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3/datalad/__init__.py\", line 17, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3/datalad/cmd.py\", line 35, in <module>\n    from .utils import on_windows, get_tempfile_kwargs, assure_unicode\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3/datalad/utils.py\", line 24, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 288ms\nInstalled 26 packages in 3.28s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 88 packages in 1.31s\nPrepared 1 package in 1.31s\nInstalled 77 packages in 161ms\n + appdirs==1.4.4\n + attrs==24.2.0\n + automat==22.10.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + constantly==15.1.0\n + cryptography==44.0.0\n + cssselect==1.2.0\n + datalad==0.9.1.dev1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3)\n + deprecated==1.2.15\n + duecredit==0.9.3\n + exifread==3.0.0\n + filelock==3.12.2\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + hyperlink==21.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + incremental==22.10.0\n + iso8601==2.1.0\n + itemadapter==0.8.0\n + itemloaders==1.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jmespath==1.0.1\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack-python==0.5.6\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + parsel==1.8.1\n + patool==1.12\n + pillow==9.5.0\n + protego==0.3.0\n + pyasn1==0.5.1\n + pyasn1-modules==0.3.0\n + pycparser==2.21\n + pydicom==2.4.4\n + pydispatcher==2.0.7\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyopenssl==24.3.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + queuelib==1.6.2\n + requests==2.31.0\n + requests-file==2.1.0\n + requests-ftp==0.3.1\n + scrapy==2.9.0\n + secretstorage==3.3.3\n + service-identity==21.1.0\n + simplejson==3.19.3\n + smmap==5.0.1\n + soupsieve==2.4.1\n + tldextract==4.0.0\n + tqdm==4.67.1\n + twisted==23.8.0\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + w3lib==2.1.2\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\n + zope-interface==6.4.post2\nResolved 107 packages in 274ms\nInstalled 19 packages in 104ms\n + alabaster==0.7.13\n + babel==2.14.0\n + docutils==0.19\n + imagesize==1.4.1\n + jinja2==3.1.5\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + pygments==2.17.2\n + pypandoc==1.14\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3\nplugins: cov-4.1.0\ncollected 27 items\n\nr2e_tests/test_1.py ....................FFFFFFF                          [100%]\n\n=================================== FAILURES ===================================\n__________________________ test_addurls_nonannex_repo __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex95th6x7k'}\nfilename = '/tmp/datalad_temp_test_addurls_nonannex_repobs448rnu'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_nonannex_repo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n_____________________________ test_addurls_dry_run _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexhh_6fflx'}\nfilename = '/tmp/datalad_temp_test_addurls_dry_runzczuhcy5'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dry_run() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n___________________________ TestAddurls.test_addurls ___________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x734f7370d390>,)\nkw = {'path': '/tmp/tmp-page2annex2sbuse25'}\nfilename = '/tmp/datalad_temp_test_addurlsqh6j1cwg'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n__________________ TestAddurls.test_addurls_create_newdataset __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x734f7370d5d0>,)\nkw = {'path': '/tmp/tmp-page2annexw52di7cb'}\nfilename = '/tmp/datalad_temp_test_addurls_create_newdatasetu2qu9m3j'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_create_newdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n_____________________ TestAddurls.test_addurls_subdataset ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x734f7370d7d0>,)\nkw = {'path': '/tmp/tmp-page2annexy83j5yjc'}\nfilename = '/tmp/datalad_temp_test_addurls_subdatasetp69lzrgy'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_subdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n______________________ TestAddurls.test_addurls_repindex _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x734f7370d9d0>,)\nkw = {'path': '/tmp/tmp-page2annexhe_i9tfl'}\nfilename = '/tmp/datalad_temp_test_addurls_repindexjlgynrs9'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_repindex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n______________________ TestAddurls.test_addurls_url_parts ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x734f736f8a90>,)\nkw = {'path': '/tmp/tmp-page2annex2lozojac'}\nfilename = '/tmp/datalad_temp_test_addurls_url_parts44ejp0j4'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_parts() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n==================================== PASSES ====================================\n___________________________ test_addurls_no_urlfile ____________________________\n----------------------------- Captured stderr call -----------------------------\n[ERROR] Must specify url_file argument [addurls(/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3)] \n------------------------------ Captured log call -------------------------------\nERROR    datalad.plugin:utils.py:508 Must specify url_file argument [addurls(/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3)]\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_lower_case\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_formatter_missing_arg\nPASSED r2e_tests/test_1.py::test_repformatter\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_get_subpaths\nPASSED r2e_tests/test_1.py::test_is_legal_metafield\nPASSED r2e_tests/test_1.py::test_filter_legal_metafield\nPASSED r2e_tests/test_1.py::test_fmt_to_name\nPASSED r2e_tests/test_1.py::test_get_url_names\nPASSED r2e_tests/test_1.py::test_extract\nPASSED r2e_tests/test_1.py::test_extract_disable_autometa\nPASSED r2e_tests/test_1.py::test_extract_exclude_autometa_regexp\nPASSED r2e_tests/test_1.py::test_extract_csv_json_equal\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\nPASSED r2e_tests/test_1.py::test_addurls_no_urlfile\nFAILED r2e_tests/test_1.py::test_addurls_nonannex_repo - TypeError: test_addu...\nFAILED r2e_tests/test_1.py::test_addurls_dry_run - TypeError: test_addurls_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls - TypeError: test_addur...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_create_newdataset - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_subdataset - TypeError:...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_repindex - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_parts - TypeError: ...\n========================= 7 failed, 20 passed in 1.56s =========================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3\nplugins: cov-4.1.0\ncollected 27 items\n\nr2e_tests/test_1.py ................F...FFFFFFF                          [100%]\n\n=================================== FAILURES ===================================\n_____________________ test_extract_exclude_autometa_regexp _____________________\n\n    def test_extract_exclude_autometa_regexp():\n        info, subpaths = addurls.extract(\n            json_stream(ST_DATA[\"rows\"]), \"json\",\n            \"{age_group}//{now_dead}//{name}.csv\",\n            \"{name}_{debut_season}.com\",\n            \"ea\",\n            [],\n            None)\n    \n        eq_([set(d[\"meta_args\"]) for d in info],\n            [{\"name=will\", \"age_group=kid\"},\n             {\"name=bob\", \"age_group=adult\"},\n             {\"name=scott\", \"age_group=adult\"},\n>            {\"name=max\", \"age_group=kid\"}])\n\nr2e_tests/test_1.py:240: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = [set(), set(), set(), set()]\nb = [{'age_group=kid', 'name=will'}, {'age_group=adult', 'name=bob'}, {'age_group=adult', 'name=scott'}, {'age_group=kid', 'name=max'}]\nmsg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: [set(), set(), set(), set()] != [{'age_group=kid', 'name=will'}, {'age_group=adult', 'name=bob'}, {'age_group=adult', 'name=scott'}, {'name=max', 'age_group=kid'}]\n\n.venv/lib/python3.7/site-packages/nose/tools/trivial.py:29: AssertionError\n__________________________ test_addurls_nonannex_repo __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexqxv1n5xz'}\nfilename = '/tmp/datalad_temp_test_addurls_nonannex_repoep2zjs7i'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_nonannex_repo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n_____________________________ test_addurls_dry_run _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexx7x9zwpv'}\nfilename = '/tmp/datalad_temp_test_addurls_dry_run13hwn8_x'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dry_run() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n___________________________ TestAddurls.test_addurls ___________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x71c3f5e92310>,)\nkw = {'path': '/tmp/tmp-page2annex9w4nn_vl'}\nfilename = '/tmp/datalad_temp_test_addurlsu10wdter'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n__________________ TestAddurls.test_addurls_create_newdataset __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x71c3f5e92550>,)\nkw = {'path': '/tmp/tmp-page2annexi7p3lgn5'}\nfilename = '/tmp/datalad_temp_test_addurls_create_newdatasetclpm9c9b'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_create_newdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n_____________________ TestAddurls.test_addurls_subdataset ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x71c3f5e92750>,)\nkw = {'path': '/tmp/tmp-page2annexa2e1_8c1'}\nfilename = '/tmp/datalad_temp_test_addurls_subdatasetlzoq3ppd'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_subdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n______________________ TestAddurls.test_addurls_repindex _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x71c3f5e92950>,)\nkw = {'path': '/tmp/tmp-page2annexi2_xcazn'}\nfilename = '/tmp/datalad_temp_test_addurls_repindex50skkawm'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_repindex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n______________________ TestAddurls.test_addurls_url_parts ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x71c3f5e92b50>,)\nkw = {'path': '/tmp/tmp-page2annexrd2mpnx0'}\nfilename = '/tmp/datalad_temp_test_addurls_url_partsz4einarn'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_parts() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n==================================== PASSES ====================================\n___________________________ test_addurls_no_urlfile ____________________________\n----------------------------- Captured stderr call -----------------------------\n[ERROR] Must specify url_file argument [addurls(/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3)] \n------------------------------ Captured log call -------------------------------\nERROR    datalad.plugin:utils.py:508 Must specify url_file argument [addurls(/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4476540ed403fd0dea8e1d717ff85ae1d56ae6f3)]\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_lower_case\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_formatter_missing_arg\nPASSED r2e_tests/test_1.py::test_repformatter\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_get_subpaths\nPASSED r2e_tests/test_1.py::test_is_legal_metafield\nPASSED r2e_tests/test_1.py::test_filter_legal_metafield\nPASSED r2e_tests/test_1.py::test_fmt_to_name\nPASSED r2e_tests/test_1.py::test_get_url_names\nPASSED r2e_tests/test_1.py::test_extract\nPASSED r2e_tests/test_1.py::test_extract_disable_autometa\nPASSED r2e_tests/test_1.py::test_extract_csv_json_equal\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\nPASSED r2e_tests/test_1.py::test_addurls_no_urlfile\nFAILED r2e_tests/test_1.py::test_extract_exclude_autometa_regexp - AssertionE...\nFAILED r2e_tests/test_1.py::test_addurls_nonannex_repo - TypeError: test_addu...\nFAILED r2e_tests/test_1.py::test_addurls_dry_run - TypeError: test_addurls_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls - TypeError: test_addur...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_create_newdataset - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_subdataset - TypeError:...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_repindex - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_parts - TypeError: ...\n========================= 8 failed, 19 passed in 0.91s =========================\n",
    "old_commit_res_stderr": ""
}
datalad/plugin/addurls.py datalad/plugin/tests/test_addurls.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Automatic Metadata Extraction Fails to Exclude Columns Using Regular Expressions

**Description:**
When attempting to exclude certain columns from automatic metadata extraction using regular expressions, the system does not properly exclude the specified columns. This results in metadata being incorrectly included or entirely missing.

**Example Code:**
```python
from datalad.plugin import addurls

def test_metadata_exclusion():
    stream = json_stream(ST_DATA['rows'])
    input_type = 'json'
    filename_format = "{age_group}//{now_dead}//{name}.csv"
    url_format = "{name}_{debut_season}.com"
    exclude_autometa = "ea"  # Intended to exclude columns matching the regex "ea"
    meta = []
    missing_value = None

    info, subpaths = addurls.extract(
        stream, input_type, filename_format, url_format,
        exclude_autometa, meta, missing_value
    )

    # Expected metadata arguments should exclude columns matching "ea"
    expected_meta_args = [
        {"name=will", "age_group=kid"},
        {"name=bob", "age_group=adult"},
        {"name=scott", "age_group=adult"},
        {"name=max", "age_group=kid"}
    ]

    # The actual metadata arguments are unexpectedly empty
    actual_meta_args = [set(d["meta_args"]) for d in info]
    print(actual_meta_args)
```

**Expected Behavior:**
Columns that match the regular expression provided in `exclude_autometa` should be excluded from the automatic metadata extraction. In the example above, any column name containing "ea" should not be included in the `meta_args`. The expected `meta_args` should contain the specified metadata pairs without the excluded columns.

**Actual Behavior:**
Despite specifying a regular expression to exclude certain columns, the `meta_args` list remains empty. This indicates that the exclusion is not being applied, leading to missing or incorrect metadata.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import re
import string

from six import string_types
from six.moves.urllib.parse import urlparse

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    missing : str, optional
        When column lookup results in an empty string, use this value in
        its place.
    """

    def __init__(self, idx_to_name, missing_value=None, *args, **kwargs):
        self.idx_to_name = idx_to_name or {}
        self.missing = missing_value
        super(Formatter, self).__init__(*args, **kwargs)

    def format(self, format_string, *args, **kwargs):
        if not isinstance(args[0], Mapping):
            raise ValueError("First positional argument should be mapping")
        return super(Formatter, self).format(format_string, *args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.


        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]

        name = key
        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            name = self.idx_to_name[key_int]

        try:
            value = data[name]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)

        if self.missing is not None and isinstance(value, string_types):
            return value or self.missing
        return value

    def convert_field(self, value, conversion):
        if conversion == 'l':
            return str(value).lower()
        return super(Formatter, self).convert_field(value, conversion)


class RepFormatter(Formatter):
    """Extend Formatter to support a {_repindex} placeholder.
    """

    def __init__(self, *args, **kwargs):
        super(RepFormatter, self).__init__(*args, **kwargs)
        self.repeats = {}
        self.repindex = 0

    def format(self, *args, **kwargs):
        self.repindex = 0
        result = super(RepFormatter, self).format(*args, **kwargs)
        if result in self.repeats:
            self.repindex = self.repeats[result] + 1
            self.repeats[result] = self.repindex
            result = super(RepFormatter, self).format(*args, **kwargs)
        else:
            self.repeats[result] = 0
        return result

    def get_value(self, key, args, kwargs):
        args[0]["_repindex"] = self.repindex
        return super(RepFormatter, self).get_value(key, args, kwargs)


def clean_meta_args(args):
    """Prepare formatted metadata arguments to be passed to git-annex.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    Generator that yields processed arguments (str).
    """
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            raise ValueError("meta argument isn't in 'field=value' format")

        if not value:
            # The `url_file` may have an empty value.
            continue

        yield field + "=" + value


def get_subpaths(filename):
    """Convert "//" marker in `filename` to a list of subpaths.

    >>> get_subpaths("p1/p2//p3/p4//file")
    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])

    Note: With Python 3, the subpaths could be generated with

        itertools.accumulate(filename.split("//")[:-1], os.path.join)

    Parameters
    ----------
    filename : str
        File name with "//" marking subpaths.

    Returns
    -------
    A tuple of the filename with any "//" collapsed to a single
    separator and a list of subpaths (str).
    """
    if "//" not in filename:
        return filename, []

    spaths = []
    for part in filename.split("//")[:-1]:
        path = os.path.join(*(spaths + [part]))
        spaths.append(path)
    return filename.replace("//", os.path.sep), spaths


def is_legal_metafield(name):
    """Test whether `name` is a valid metadata field.

    The set of permitted characters is taken from git-annex's
    MetaData.hs:legalField.
    """
    legal = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z")
    return True if legal.match(name) else False


def filter_legal_metafield(fields):
    """Remove illegal names from `fields`.
    """
    legal = []
    for field in fields:
        if is_legal_metafield(field):
            legal.append(field)
        else:
            lgr.debug("%s is not a valid metadata field name; dropping",
                      field)
    return legal


def get_fmt_names(format_string):
    """Yield field names in `format_string`.
    """
    for _, name, _, _ in string.Formatter().parse(format_string):
        if name:
            yield name


def fmt_to_name(format_string, num_to_name):
    """Try to map a format string to a single name.

    Parameters
    ----------
    format_string : string
    num_to_name : dict
        A dictionary that maps from an integer to a column name.  This
        enables mapping the format string to an integer to a name.

    Returns
    -------
    A placeholder name if `format_string` consists of a single
    placeholder and no other text.  Otherwise, None is returned.
    """
    parsed = list(string.Formatter().parse(format_string))
    if len(parsed) != 1:
        # It's an empty string or there's more than one placeholder.
        return
    if parsed[0][0]:
        # Format string contains text before the placeholder.
        return

    name = parsed[0][1]
    if not name:
        # The field name is empty.
        return

    try:
        return num_to_name[int(name)]
    except (KeyError, ValueError):
        return name


def _read(stream, input_type):
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        idx_map = dict(enumerate(headers))
        rows = [dict(zip(headers, r)) for r in csvrows]
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        idx_map = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")
    return rows, idx_map


def _format_filenames(format_fn, rows, row_infos):
    subpaths = set()
    for row, info in zip(rows, row_infos):
        filename = format_fn(row)
        filename, spaths = get_subpaths(filename)
        subpaths |= set(spaths)
        info["filename"] = filename
        info["subpath"] = spaths[-1] if spaths else None
    return subpaths


def get_url_names(url):
    """Assign a name to each part of a URL's path.

    Parameters
    ----------
    url : str

    Returns
    -------
    A dict with keys '_url0' through '_urlN' for a path with N+1
    components.  There is also a `_url_basename` for the rightmost
    entry.
    """
    path = urlparse(url).path.strip("/")
    if not path:
        return {}

    url_parts = path.split("/")
    names = {}
    for pidx, part in enumerate(url_parts):
        names["_url{}".format(pidx)] = part
    names["_url_basename"] = url_parts[-1]
    return names


def extract(stream, input_type, filename_format, url_format,
            no_autometa, meta, missing_value):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.

    All other parameters match those described in `dlplugin`.

    Returns
    -------
    A tuple where the first item is a list with a dict of extracted information
    for each row in `stream` and the second item is a set that contains all the
    subdataset paths.
    """
    rows, colidx_to_name = _read(stream, input_type)

    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta
    format_url = partial(fmt.format, url_format)

    auto_meta_args = []
    if not no_autometa:
        urlcol = fmt_to_name(url_format, colidx_to_name)
        # TODO: Try to normalize invalid fields, checking for any
        # collisions.
        #
        # TODO: Add a command line argument for excluding a particular
        # column(s), maybe based on a regexp.
        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())
                                          if c != urlcol)
        auto_meta_args = [c + "=" + "{" + c + "}" for c in metacols]

    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]

    infos = []
    for row in rows:
        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))
        infos.append({"url": format_url(row),
                      "meta_args": meta_args})

    # Format the filename in a second pass so that we can provide
    # information about the formatted URLs.
    if any(i.startswith("_url") for i in get_fmt_names(filename_format)):
        for row, info in zip(rows, infos):
            row.update(get_url_names(info["url"]))

    # For the file name, we allow the _repindex special key.
    format_filename = partial(
        RepFormatter(colidx_to_name, missing_value).format,
        filename_format)
    subpaths = _format_filenames(format_filename, rows, infos)
    return infos, subpaths


def dlplugin(dataset=None, url_file=None, input_type="ext",
             url_format="{0}", filename_format="{1}",
             no_autometa=False, meta=None,
             message=None, dry_run=False, fast=False,
             ifexists=None, missing_value=None):
    """Create and update a dataset from a list of URLs.

    Parameters
    ----------
    dataset : Dataset
        Add the URLs to this dataset (or possibly subdatasets of this
        dataset).  An empty or non-existent directory is passed to
        create a new dataset.  New subdatasets can be specified with
        `filename_format`.
    url_file : str
        A file that contains URLs or information that can be used to
        construct URLs.  Depending on the value of `input_type`, this
        should be a CSV file (with a header as the first row) or a
        JSON file (structured as a list of objects with string
        values).
    input_type : {"ext", "csv", "json"}, optional
        Whether `url_file` should be considered a CSV file or a JSON
        file.  The default value, "ext", means to consider `url_file`
        as a JSON file if it ends with ".json".  Otherwise, treat it
        as a CSV file.
    url_format : str, optional
        A format string that specifies the URL for each entry.  This
        value is similar to a normal Python format string where the
        names from `url_file` (column names for a CSV or properties
        for JSON) are available as placeholders.  If `url_file` is a
        CSV file, a positional index can also be used (i.e., "{0}" for
        the first column).  Note that a placeholder cannot contain a
        ':' or '!'.
    filename_format : str, optional
        Like `url_format`, but this format string specifies the file to
        which the URL's content will be downloaded.  The file name may
        contain directories.  The separator "//" can be used to indicate
        that the left-side directory should be created as a new
        subdataset.

        In addition to the placeholders described in `url_format`, there
        are a few special placeholder.

          - "_repindex"

            The constructed file names must be unique across all fields
            rows.  To avoid collisions, the special placeholder
            "_repindex" can added to the formatter.  It value will start
            at 0 and increment every time a file name repeats.

          - "_urlN"

            Each part of the formatted URL is available.  For example,
            in "http://datalad.org/for/git-users", "_url0" and "_url1"
            would map to "for" and "git-users", respectively.

            The final part is also available as "_url_basename".
    no_autometa : bool, optional
        By default, metadata field=value pairs are constructed with each
        column in `url_file`, excluding any single column that is
        specified via `url_format`.  Set this flag to True to disable
        this behavior.  Metadata can still be set explicitly with the
        `meta` argument.
    meta : str, optional
        A format string that specifies metadata.  It should be
        structured as "<field>=<value>".  The same placeholders from
        `url_format` can be used.  As an example, "location={3}" would
        mean that the value for the "location" metadata field should be
        set the value of the fourth column.  This option can be given
        multiple times.
    message : str, optional
        Use this message when committing the URL additions.
    dry_run : bool, optional
        Report which URLs would be downloaded to which files and then
        exit.
    fast : bool, optional
        If True, add the URLs, but don't download their content.
        Underneath, this passes the --fast flag to `git annex addurl`.
    ifexists : {None, 'overwrite', 'skip'}
        What to do if a constructed file name already exists.  The
        default (None) behavior to proceed with the `git annex addurl`,
        which will failed if the file size has changed.  If set to
        'overwrite', remove the old file before adding the new one.  If
        set to 'skip', do not add the new file.
    missing_value : str, optional
        When an empty string is encountered, use this value instead.

    Examples
    --------
    Consider a file "avatars.csv" that contains

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and
    'ext' fields, we could run

        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv
          url_format='{link}' filename_format='{who}.{ext}' fast=True

    The '-d avatar_ds' is used to create a new dataset in
    "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new
    subdataset in an "avatars" subdirectory, we could use "//" in the
    `filename_format` argument:

        $ datalad plugin addurls url_file=avatars.csv
          url_format='{link}' filename_format='avatars//{who}.{ext}'
          fast=True

    Note
    ----
    For users familiar with 'git annex addurl': A large part of this
    plugin's functionality can be viewed as transforming data from
    `url_file` into a "url filename" format that fed to 'git annex
    addurl --batch --with-files'.
    """
    import logging
    import os

    from datalad.distribution.dataset import Dataset
    from datalad.interface.results import get_status_dict
    import datalad.plugin.addurls as me
    from datalad.support.annexrepo import AnnexRepo
    from datalad.support.exceptions import AnnexBatchCommandError
    from datalad.utils import assure_list
    from datalad.ui import ui

    lgr = logging.getLogger("datalad.plugin.addurls")

    meta = assure_list(meta)

    if url_file is None:
        # `url_file` is not a required argument in `dlplugin` because
        # the argument before it, `dataset`, needs to be optional to
        # support the creation of new datasets.
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="Must specify url_file argument")
        return

    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="not an annex repo")
        return

    if input_type == "ext":
        extension = os.path.splitext(url_file)[1]
        input_type = "json" if extension == ".json" else "csv"

    with open(url_file) as fd:
        rows, subpaths = me.extract(fd, input_type,
                                    filename_format, url_format,
                                    no_autometa, meta,
                                    missing_value)

    all_files = [row["filename"] for row in rows]
    if len(all_files) != len(set(all_files)):
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message=("There are file name collisions; "
                                       "consider using {_repindex}"))
        return

    if dry_run:
        for subpath in subpaths:
            lgr.info("Would create a subdataset at %s", subpath)
        for row in rows:
            lgr.info("Would download %s to %s",
                     row["url"], os.path.join(dataset.path, row["filename"]))
            lgr.info("Metadata: %s", row["meta_args"])
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="ok",
                              message="dry-run finished")
        return

    if not dataset.repo:
        # Populate a new dataset with the URLs.
        for r in dataset.create(result_xfm=None, return_type='generator'):
            yield r

    annex_options = ["--fast"] if fast else []

    for spath in subpaths:
        if os.path.exists(os.path.join(dataset.path, spath)):
            lgr.warning(
                "Not creating subdataset at existing path: %s",
                spath)
        else:
            for r in dataset.create(spath, result_xfm=None,
                                    return_type='generator'):
                yield r

    pbar_addurl = ui.get_progressbar(total=len(rows), label="Adding files",
                                     unit=" Files")
    pbar_addurl.start()

    files_to_add = []
    meta_to_add = []
    addurl_results = []
    for row_idx, row in enumerate(rows, 1):
        pbar_addurl.update(row_idx)
        fname_abs = os.path.join(dataset.path, row["filename"])
        if row["subpath"]:
            # Adjust the dataset and filename for an `addurl` call
            # from within the subdataset that will actually contain
            # the link.
            ds_current = Dataset(os.path.join(dataset.path, row["subpath"]))
            ds_filename = os.path.relpath(fname_abs, ds_current.path)
        else:
            ds_current = dataset
            ds_filename = row["filename"]

        if os.path.exists(fname_abs) or os.path.islink(fname_abs):
            if ifexists == "skip":
                addurl_results.append(
                    get_status_dict(action="addurls",
                                    ds=ds_current,
                                    type="file",
                                    path=fname_abs,
                                    status="notneeded"))
                continue
            elif ifexists == "overwrite":
                lgr.debug("Removing %s", fname_abs)
                os.unlink(fname_abs)
            else:
                lgr.debug("File %s already exists", fname_abs)

        try:
            ds_current.repo.add_url_to_file(ds_filename, row["url"],
                                            batch=True, options=annex_options)
        except AnnexBatchCommandError:
            addurl_results.append(
                get_status_dict(action="addurls",
                                ds=ds_current,
                                type="file",
                                path=os.path.join(ds_current.path,
                                                  ds_filename),
                                message="failed to add URL",
                                status="error"))

            continue
        # Collect the status dicts to yield later so that we don't
        # interrupt the progress bar.
        addurl_results.append(
            get_status_dict(action="addurls",
                            ds=ds_current,
                            type="file",
                            path=os.path.join(ds_current.path,
                                              ds_filename),
                            status="ok"))

        files_to_add.append(row["filename"])
        meta_to_add.append((ds_current, ds_filename, row["meta_args"]))
    pbar_addurl.finish()

    for result in addurl_results:
        yield result

        msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)

    if files_to_add:
        for r in dataset.add(files_to_add, message=msg):
            yield r

    # TODO: Wrap this repeated "delayed results/progress bar" pattern.
    pbar_meta = ui.get_progressbar(total=len(meta_to_add),
                                   label="Adding metadata", unit=" Files")
    pbar_meta.start()

    meta_results = []
    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):
        pbar_meta.update(meta_idx)
        lgr.debug("Adding metadata to %s in %s", fname, ds.path)
        for arg in meta:
            ds.repo._run_annex_command("metadata",
                                       annex_options=["--set", arg, fname])
        meta_results.append(
            get_status_dict(action="addurls-metadata",
                            ds=ds_current,
                            type="file",
                            path=os.path.join(ds.path, fname),
                            message="added metadata",
                            status="ok"))

    for result in meta_results:
        yield result

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Automatic Metadata Extraction Fails to Exclude Columns Using Regular Expressions

**Description:**
When attempting to exclude certain columns from automatic metadata extraction using regular expressions, the system does not properly exclude the specified columns. This results in metadata being incorrectly included or entirely missing.

**Example Code:**
```python
from datalad.plugin import addurls

def test_metadata_exclusion():
    stream = json_stream(ST_DATA['rows'])
    input_type = 'json'
    filename_format = "{age_group}//{now_dead}//{name}.csv"
    url_format = "{name}_{debut_season}.com"
    exclude_autometa = "ea"  # Intended to exclude columns matching the regex "ea"
    meta = []
    missing_value = None

    info, subpaths = addurls.extract(
        stream, input_type, filename_format, url_format,
        exclude_autometa, meta, missing_value
    )

    # Expected metadata arguments should exclude columns matching "ea"
    expected_meta_args = [
        {"name=will", "age_group=kid"},
        {"name=bob", "age_group=adult"},
        {"name=scott", "age_group=adult"},
        {"name=max", "age_group=kid"}
    ]

    # The actual metadata arguments are unexpectedly empty
    actual_meta_args = [set(d["meta_args"]) for d in info]
    print(actual_meta_args)
```

**Expected Behavior:**
Columns that match the regular expression provided in `exclude_autometa` should be excluded from the automatic metadata extraction. In the example above, any column name containing "ea" should not be included in the `meta_args`. The expected `meta_args` should contain the specified metadata pairs without the excluded columns.

**Actual Behavior:**
Despite specifying a regular expression to exclude certain columns, the `meta_args` list remains empty. This indicates that the exclusion is not being applied, leading to missing or incorrect metadata.

[/ISSUE]
{
    "test_formatter": "PASSED",
    "test_formatter_lower_case": "PASSED",
    "test_formatter_no_idx_map": "PASSED",
    "test_formatter_no_mapping_arg": "PASSED",
    "test_formatter_placeholder_with_spaces": "PASSED",
    "test_formatter_placeholder_nonpermitted_chars": "PASSED",
    "test_formatter_missing_arg": "PASSED",
    "test_repformatter": "PASSED",
    "test_clean_meta_args": "PASSED",
    "test_get_subpaths": "PASSED",
    "test_is_legal_metafield": "PASSED",
    "test_filter_legal_metafield": "PASSED",
    "test_fmt_to_name": "PASSED",
    "test_get_url_names": "PASSED",
    "test_extract": "PASSED",
    "test_extract_disable_autometa": "PASSED",
    "test_extract_exclude_autometa_regexp": "PASSED",
    "test_extract_csv_json_equal": "PASSED",
    "test_extract_wrong_input_type": "PASSED",
    "test_addurls_no_urlfile": "PASSED",
    "test_addurls_nonannex_repo": "FAILED",
    "test_addurls_dry_run": "FAILED",
    "TestAddurls.test_addurls": "FAILED",
    "TestAddurls.test_addurls_create_newdataset": "FAILED",
    "TestAddurls.test_addurls_subdataset": "FAILED",
    "TestAddurls.test_addurls_repindex": "FAILED",
    "TestAddurls.test_addurls_url_parts": "FAILED"
}
diff --git a/datalad/plugin/addurls.py b/datalad/plugin/addurls.py
index 8f5ddebc8..a58191fae 100644
--- a/datalad/plugin/addurls.py
+++ b/datalad/plugin/addurls.py
@@ -289,7 +289,7 @@ def get_url_names(url):
 
 
 def extract(stream, input_type, filename_format, url_format,
-            no_autometa, meta, missing_value):
+            exclude_autometa, meta, missing_value):
     """Extract and format information from `url_file`.
 
     Parameters
@@ -311,15 +311,15 @@ def extract(stream, input_type, filename_format, url_format,
     format_url = partial(fmt.format, url_format)
 
     auto_meta_args = []
-    if not no_autometa:
+    if exclude_autometa not in ["*", ""]:
         urlcol = fmt_to_name(url_format, colidx_to_name)
         # TODO: Try to normalize invalid fields, checking for any
         # collisions.
-        #
-        # TODO: Add a command line argument for excluding a particular
-        # column(s), maybe based on a regexp.
-        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())
-                                          if c != urlcol)
+        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)
+        if exclude_autometa:
+            metacols = (c for c in metacols
+                        if not re.search(exclude_autometa, c))
+        metacols = filter_legal_metafield(metacols)
         auto_meta_args = [c + "=" + "{" + c + "}" for c in metacols]
 
     # Unlike `filename_format` and `url_format`, `meta` is a list
@@ -348,7 +348,7 @@ def extract(stream, input_type, filename_format, url_format,
 
 def dlplugin(dataset=None, url_file=None, input_type="ext",
              url_format="{0}", filename_format="{1}",
-             no_autometa=False, meta=None,
+             exclude_autometa=None, meta=None,
              message=None, dry_run=False, fast=False,
              ifexists=None, missing_value=None):
     """Create and update a dataset from a list of URLs.
@@ -403,12 +403,14 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
             would map to "for" and "git-users", respectively.
 
             The final part is also available as "_url_basename".
-    no_autometa : bool, optional
+    exclude_autometa : str, optional
         By default, metadata field=value pairs are constructed with each
         column in `url_file`, excluding any single column that is
-        specified via `url_format`.  Set this flag to True to disable
-        this behavior.  Metadata can still be set explicitly with the
-        `meta` argument.
+        specified via `url_format`.  This argument can be used to
+        exclude columns that match a regular expression.  If set to '*'
+        or an empty string, automatic metadata extraction is disabled
+        completely.  This argument does not affect metadata set
+        explicitly with the `meta` argument.
     meta : str, optional
         A format string that specifies metadata.  It should be
         structured as "<field>=<value>".  The same placeholders from
@@ -504,7 +506,7 @@ def dlplugin(dataset=None, url_file=None, input_type="ext",
     with open(url_file) as fd:
         rows, subpaths = me.extract(fd, input_type,
                                     filename_format, url_format,
-                                    no_autometa, meta,
+                                    exclude_autometa, meta,
                                     missing_value)
 
     all_files = [row["filename"] for row in rows]

{"datalad/plugin/addurls.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport re\nimport string\n\nfrom six import string_types\nfrom six.moves.urllib.parse import urlparse\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name, missing_value=None, *args, **kwargs):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n        super(Formatter, self).__init__(*args, **kwargs)\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n\n\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, string_types):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Prepare formatted metadata arguments to be passed to git-annex.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    Generator that yields processed arguments (str).\n    \"\"\"\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n\n        yield field + \"=\" + value\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(*(spaths + [part]))\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    legal = re.compile(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\")\n    return True if legal.match(name) else False\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\ndef _read(stream, input_type):\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n    return rows, idx_map\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        filename = format_fn(row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_url_names(url):\n    \"\"\"Assign a name to each part of a URL's path.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys '_url0' through '_urlN' for a path with N+1\n    components.  There is also a `_url_basename` for the rightmost\n    entry.\n    \"\"\"\n    path = urlparse(url).path.strip(\"/\")\n    if not path:\n        return {}\n\n    url_parts = path.split(\"/\")\n    names = {}\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    names[\"_url_basename\"] = url_parts[-1]\n    return names\n\n\ndef extract(stream, input_type, filename_format, url_format,\n            no_autometa, meta, missing_value):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n\n    All other parameters match those described in `dlplugin`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if not no_autometa:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        #\n        # TODO: Add a command line argument for excluding a particular\n        # column(s), maybe based on a regexp.\n        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())\n                                          if c != urlcol)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    infos = []\n    for row in rows:\n        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))\n        infos.append({\"url\": format_url(row),\n                      \"meta_args\": meta_args})\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    if any(i.startswith(\"_url\") for i in get_fmt_names(filename_format)):\n        for row, info in zip(rows, infos):\n            row.update(get_url_names(info[\"url\"]))\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows, infos)\n    return infos, subpaths\n\n\ndef dlplugin(dataset=None, url_file=None, input_type=\"ext\",\n             url_format=\"{0}\", filename_format=\"{1}\",\n             no_autometa=False, meta=None,\n             message=None, dry_run=False, fast=False,\n             ifexists=None, missing_value=None):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Add the URLs to this dataset (or possibly subdatasets of this\n        dataset).  An empty or non-existent directory is passed to\n        create a new dataset.  New subdatasets can be specified with\n        `filename_format`.\n    url_file : str\n        A file that contains URLs or information that can be used to\n        construct URLs.  Depending on the value of `input_type`, this\n        should be a CSV file (with a header as the first row) or a\n        JSON file (structured as a list of objects with string\n        values).\n    input_type : {\"ext\", \"csv\", \"json\"}, optional\n        Whether `url_file` should be considered a CSV file or a JSON\n        file.  The default value, \"ext\", means to consider `url_file`\n        as a JSON file if it ends with \".json\".  Otherwise, treat it\n        as a CSV file.\n    url_format : str, optional\n        A format string that specifies the URL for each entry.  This\n        value is similar to a normal Python format string where the\n        names from `url_file` (column names for a CSV or properties\n        for JSON) are available as placeholders.  If `url_file` is a\n        CSV file, a positional index can also be used (i.e., \"{0}\" for\n        the first column).  Note that a placeholder cannot contain a\n        ':' or '!'.\n    filename_format : str, optional\n        Like `url_format`, but this format string specifies the file to\n        which the URL's content will be downloaded.  The file name may\n        contain directories.  The separator \"//\" can be used to indicate\n        that the left-side directory should be created as a new\n        subdataset.\n\n        In addition to the placeholders described in `url_format`, there\n        are a few special placeholder.\n\n          - \"_repindex\"\n\n            The constructed file names must be unique across all fields\n            rows.  To avoid collisions, the special placeholder\n            \"_repindex\" can added to the formatter.  It value will start\n            at 0 and increment every time a file name repeats.\n\n          - \"_urlN\"\n\n            Each part of the formatted URL is available.  For example,\n            in \"http://datalad.org/for/git-users\", \"_url0\" and \"_url1\"\n            would map to \"for\" and \"git-users\", respectively.\n\n            The final part is also available as \"_url_basename\".\n    no_autometa : bool, optional\n        By default, metadata field=value pairs are constructed with each\n        column in `url_file`, excluding any single column that is\n        specified via `url_format`.  Set this flag to True to disable\n        this behavior.  Metadata can still be set explicitly with the\n        `meta` argument.\n    meta : str, optional\n        A format string that specifies metadata.  It should be\n        structured as \"<field>=<value>\".  The same placeholders from\n        `url_format` can be used.  As an example, \"location={3}\" would\n        mean that the value for the \"location\" metadata field should be\n        set the value of the fourth column.  This option can be given\n        multiple times.\n    message : str, optional\n        Use this message when committing the URL additions.\n    dry_run : bool, optional\n        Report which URLs would be downloaded to which files and then\n        exit.\n    fast : bool, optional\n        If True, add the URLs, but don't download their content.\n        Underneath, this passes the --fast flag to `git annex addurl`.\n    ifexists : {None, 'overwrite', 'skip'}\n        What to do if a constructed file name already exists.  The\n        default (None) behavior to proceed with the `git annex addurl`,\n        which will failed if the file size has changed.  If set to\n        'overwrite', remove the old file before adding the new one.  If\n        set to 'skip', do not add the new file.\n    missing_value : str, optional\n        When an empty string is encountered, use this value instead.\n\n    Examples\n    --------\n    Consider a file \"avatars.csv\" that contains\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and\n    'ext' fields, we could run\n\n        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv\n          url_format='{link}' filename_format='{who}.{ext}' fast=True\n\n    The '-d avatar_ds' is used to create a new dataset in\n    \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new\n    subdataset in an \"avatars\" subdirectory, we could use \"//\" in the\n    `filename_format` argument:\n\n        $ datalad plugin addurls url_file=avatars.csv\n          url_format='{link}' filename_format='avatars//{who}.{ext}'\n          fast=True\n\n    Note\n    ----\n    For users familiar with 'git annex addurl': A large part of this\n    plugin's functionality can be viewed as transforming data from\n    `url_file` into a \"url filename\" format that fed to 'git annex\n    addurl --batch --with-files'.\n    \"\"\"\n    import logging\n    import os\n\n    from datalad.distribution.dataset import Dataset\n    from datalad.interface.results import get_status_dict\n    import datalad.plugin.addurls as me\n    from datalad.support.annexrepo import AnnexRepo\n    from datalad.support.exceptions import AnnexBatchCommandError\n    from datalad.utils import assure_list\n    from datalad.ui import ui\n\n    lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n    meta = assure_list(meta)\n\n    if url_file is None:\n        # `url_file` is not a required argument in `dlplugin` because\n        # the argument before it, `dataset`, needs to be optional to\n        # support the creation of new datasets.\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"Must specify url_file argument\")\n        return\n\n    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=\"not an annex repo\")\n        return\n\n    if input_type == \"ext\":\n        extension = os.path.splitext(url_file)[1]\n        input_type = \"json\" if extension == \".json\" else \"csv\"\n\n    with open(url_file) as fd:\n        rows, subpaths = me.extract(fd, input_type,\n                                    filename_format, url_format,\n                                    no_autometa, meta,\n                                    missing_value)\n\n    all_files = [row[\"filename\"] for row in rows]\n    if len(all_files) != len(set(all_files)):\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"error\",\n                              message=(\"There are file name collisions; \"\n                                       \"consider using {_repindex}\"))\n        return\n\n    if dry_run:\n        for subpath in subpaths:\n            lgr.info(\"Would create a subdataset at %s\", subpath)\n        for row in rows:\n            lgr.info(\"Would download %s to %s\",\n                     row[\"url\"], os.path.join(dataset.path, row[\"filename\"]))\n            lgr.info(\"Metadata: %s\", row[\"meta_args\"])\n        yield get_status_dict(action=\"addurls\",\n                              ds=dataset,\n                              status=\"ok\",\n                              message=\"dry-run finished\")\n        return\n\n    if not dataset.repo:\n        # Populate a new dataset with the URLs.\n        for r in dataset.create(result_xfm=None, return_type='generator'):\n            yield r\n\n    annex_options = [\"--fast\"] if fast else []\n\n    for spath in subpaths:\n        if os.path.exists(os.path.join(dataset.path, spath)):\n            lgr.warning(\n                \"Not creating subdataset at existing path: %s\",\n                spath)\n        else:\n            for r in dataset.create(spath, result_xfm=None,\n                                    return_type='generator'):\n                yield r\n\n    pbar_addurl = ui.get_progressbar(total=len(rows), label=\"Adding files\",\n                                     unit=\" Files\")\n    pbar_addurl.start()\n\n    files_to_add = []\n    meta_to_add = []\n    addurl_results = []\n    for row_idx, row in enumerate(rows, 1):\n        pbar_addurl.update(row_idx)\n        fname_abs = os.path.join(dataset.path, row[\"filename\"])\n        if row[\"subpath\"]:\n            # Adjust the dataset and filename for an `addurl` call\n            # from within the subdataset that will actually contain\n            # the link.\n            ds_current = Dataset(os.path.join(dataset.path, row[\"subpath\"]))\n            ds_filename = os.path.relpath(fname_abs, ds_current.path)\n        else:\n            ds_current = dataset\n            ds_filename = row[\"filename\"]\n\n        if os.path.exists(fname_abs) or os.path.islink(fname_abs):\n            if ifexists == \"skip\":\n                addurl_results.append(\n                    get_status_dict(action=\"addurls\",\n                                    ds=ds_current,\n                                    type=\"file\",\n                                    path=fname_abs,\n                                    status=\"notneeded\"))\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", fname_abs)\n                os.unlink(fname_abs)\n            else:\n                lgr.debug(\"File %s already exists\", fname_abs)\n\n        try:\n            ds_current.repo.add_url_to_file(ds_filename, row[\"url\"],\n                                            batch=True, options=annex_options)\n        except AnnexBatchCommandError:\n            addurl_results.append(\n                get_status_dict(action=\"addurls\",\n                                ds=ds_current,\n                                type=\"file\",\n                                path=os.path.join(ds_current.path,\n                                                  ds_filename),\n                                message=\"failed to add URL\",\n                                status=\"error\"))\n\n            continue\n        # Collect the status dicts to yield later so that we don't\n        # interrupt the progress bar.\n        addurl_results.append(\n            get_status_dict(action=\"addurls\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds_current.path,\n                                              ds_filename),\n                            status=\"ok\"))\n\n        files_to_add.append(row[\"filename\"])\n        meta_to_add.append((ds_current, ds_filename, row[\"meta_args\"]))\n    pbar_addurl.finish()\n\n    for result in addurl_results:\n        yield result\n\n        msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n    if files_to_add:\n        for r in dataset.add(files_to_add, message=msg):\n            yield r\n\n    # TODO: Wrap this repeated \"delayed results/progress bar\" pattern.\n    pbar_meta = ui.get_progressbar(total=len(meta_to_add),\n                                   label=\"Adding metadata\", unit=\" Files\")\n    pbar_meta.start()\n\n    meta_results = []\n    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):\n        pbar_meta.update(meta_idx)\n        lgr.debug(\"Adding metadata to %s in %s\", fname, ds.path)\n        for arg in meta:\n            ds.repo._run_annex_command(\"metadata\",\n                                       annex_options=[\"--set\", arg, fname])\n        meta_results.append(\n            get_status_dict(action=\"addurls-metadata\",\n                            ds=ds_current,\n                            type=\"file\",\n                            path=os.path.join(ds.path, fname),\n                            message=\"added metadata\",\n                            status=\"ok\"))\n\n    for result in meta_results:\n        yield result\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Automatic Metadata Extraction Fails to Exclude Columns Using Regular Expressions

**Description:**
When attempting to exclude certain columns from automatic metadata extraction using regular expressions, the system does not properly exclude the specified columns. This results in metadata being incorrectly included or entirely missing.

**Example Code:**
```python
from datalad.plugin import addurls

def test_metadata_exclusion():
    stream = json_stream(ST_DATA['rows'])
    input_type = 'json'
    filename_format = "{age_group}//{now_dead}//{name}.csv"
    url_format = "{name}_{debut_season}.com"
    exclude_autometa = "ea"  # Intended to exclude columns matching the regex "ea"
    meta = []
    missing_value = None

    info, subpaths = addurls.extract(
        stream, input_type, filename_format, url_format,
        exclude_autometa, meta, missing_value
    )

    # Expected metadata arguments should exclude columns matching "ea"
    expected_meta_args = [
        {"name=will", "age_group=kid"},
        {"name=bob", "age_group=adult"},
        {"name=scott", "age_group=adult"},
        {"name=max", "age_group=kid"}
    ]

    # The actual metadata arguments are unexpectedly empty
    actual_meta_args = [set(d["meta_args"]) for d in info]
    print(actual_meta_args)
```

**Expected Behavior:**
Columns that match the regular expression provided in `exclude_autometa` should be excluded from the automatic metadata extraction. In the example above, any column name containing "ea" should not be included in the `meta_args`. The expected `meta_args` should contain the specified metadata pairs without the excluded columns.

**Actual Behavior:**
Despite specifying a regular expression to exclude certain columns, the `meta_args` list remains empty. This indicates that the exclusion is not being applied, leading to missing or incorrect metadata.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import re
import string

from six import string_types
from six.moves.urllib.parse import urlparse

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    missing : str, optional
        When column lookup results in an empty string, use this value in
        its place.
    """

    def __init__(self, idx_to_name, missing_value=None, *args, **kwargs):
        self.idx_to_name = idx_to_name or {}
        self.missing = missing_value
        super(Formatter, self).__init__(*args, **kwargs)

    def format(self, format_string, *args, **kwargs):
        if not isinstance(args[0], Mapping):
            raise ValueError("First positional argument should be mapping")
        return super(Formatter, self).format(format_string, *args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.


        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]

        name = key
        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            name = self.idx_to_name[key_int]

        try:
            value = data[name]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)

        if self.missing is not None and isinstance(value, string_types):
            return value or self.missing
        return value

    def convert_field(self, value, conversion):
        if conversion == 'l':
            return str(value).lower()
        return super(Formatter, self).convert_field(value, conversion)


class RepFormatter(Formatter):
    """Extend Formatter to support a {_repindex} placeholder.
    """

    def __init__(self, *args, **kwargs):
        super(RepFormatter, self).__init__(*args, **kwargs)
        self.repeats = {}
        self.repindex = 0

    def format(self, *args, **kwargs):
        self.repindex = 0
        result = super(RepFormatter, self).format(*args, **kwargs)
        if result in self.repeats:
            self.repindex = self.repeats[result] + 1
            self.repeats[result] = self.repindex
            result = super(RepFormatter, self).format(*args, **kwargs)
        else:
            self.repeats[result] = 0
        return result

    def get_value(self, key, args, kwargs):
        args[0]["_repindex"] = self.repindex
        return super(RepFormatter, self).get_value(key, args, kwargs)


def clean_meta_args(args):
    """Prepare formatted metadata arguments to be passed to git-annex.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    Generator that yields processed arguments (str).
    """
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            raise ValueError("meta argument isn't in 'field=value' format")

        if not value:
            # The `url_file` may have an empty value.
            continue

        yield field + "=" + value


def get_subpaths(filename):
    """Convert "//" marker in `filename` to a list of subpaths.

    >>> get_subpaths("p1/p2//p3/p4//file")
    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])

    Note: With Python 3, the subpaths could be generated with

        itertools.accumulate(filename.split("//")[:-1], os.path.join)

    Parameters
    ----------
    filename : str
        File name with "//" marking subpaths.

    Returns
    -------
    A tuple of the filename with any "//" collapsed to a single
    separator and a list of subpaths (str).
    """
    if "//" not in filename:
        return filename, []

    spaths = []
    for part in filename.split("//")[:-1]:
        path = os.path.join(*(spaths + [part]))
        spaths.append(path)
    return filename.replace("//", os.path.sep), spaths


def is_legal_metafield(name):
    """Test whether `name` is a valid metadata field.

    The set of permitted characters is taken from git-annex's
    MetaData.hs:legalField.
    """
    legal = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z")
    return True if legal.match(name) else False


def filter_legal_metafield(fields):
    """Remove illegal names from `fields`.
    """
    legal = []
    for field in fields:
        if is_legal_metafield(field):
            legal.append(field)
        else:
            lgr.debug("%s is not a valid metadata field name; dropping",
                      field)
    return legal


def get_fmt_names(format_string):
    """Yield field names in `format_string`.
    """
    for _, name, _, _ in string.Formatter().parse(format_string):
        if name:
            yield name


def fmt_to_name(format_string, num_to_name):
    """Try to map a format string to a single name.

    Parameters
    ----------
    format_string : string
    num_to_name : dict
        A dictionary that maps from an integer to a column name.  This
        enables mapping the format string to an integer to a name.

    Returns
    -------
    A placeholder name if `format_string` consists of a single
    placeholder and no other text.  Otherwise, None is returned.
    """
    parsed = list(string.Formatter().parse(format_string))
    if len(parsed) != 1:
        # It's an empty string or there's more than one placeholder.
        return
    if parsed[0][0]:
        # Format string contains text before the placeholder.
        return

    name = parsed[0][1]
    if not name:
        # The field name is empty.
        return

    try:
        return num_to_name[int(name)]
    except (KeyError, ValueError):
        return name


def _read(stream, input_type):
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        idx_map = dict(enumerate(headers))
        rows = [dict(zip(headers, r)) for r in csvrows]
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        idx_map = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")
    return rows, idx_map


def _format_filenames(format_fn, rows, row_infos):
    subpaths = set()
    for row, info in zip(rows, row_infos):
        filename = format_fn(row)
        filename, spaths = get_subpaths(filename)
        subpaths |= set(spaths)
        info["filename"] = filename
        info["subpath"] = spaths[-1] if spaths else None
    return subpaths


def get_url_names(url):
    """Assign a name to each part of a URL's path.

    Parameters
    ----------
    url : str

    Returns
    -------
    A dict with keys '_url0' through '_urlN' for a path with N+1
    components.  There is also a `_url_basename` for the rightmost
    entry.
    """
    path = urlparse(url).path.strip("/")
    if not path:
        return {}

    url_parts = path.split("/")
    names = {}
    for pidx, part in enumerate(url_parts):
        names["_url{}".format(pidx)] = part
    names["_url_basename"] = url_parts[-1]
    return names


def extract(stream, input_type, filename_format, url_format,
            no_autometa, meta, missing_value):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.

    All other parameters match those described in `dlplugin`.

    Returns
    -------
    A tuple where the first item is a list with a dict of extracted information
    for each row in `stream` and the second item is a set that contains all the
    subdataset paths.
    """
    rows, colidx_to_name = _read(stream, input_type)

    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta
    format_url = partial(fmt.format, url_format)

    auto_meta_args = []
    if not no_autometa:
        urlcol = fmt_to_name(url_format, colidx_to_name)
        # TODO: Try to normalize invalid fields, checking for any
        # collisions.
        #
        # TODO: Add a command line argument for excluding a particular
        # column(s), maybe based on a regexp.
        metacols = filter_legal_metafield(c for c in sorted(rows[0].keys())
                                          if c != urlcol)
        auto_meta_args = [c + "=" + "{" + c + "}" for c in metacols]

    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]

    infos = []
    for row in rows:
        meta_args = list(clean_meta_args(fmt(row) for fmt in formats_meta))
        infos.append({"url": format_url(row),
                      "meta_args": meta_args})

    # Format the filename in a second pass so that we can provide
    # information about the formatted URLs.
    if any(i.startswith("_url") for i in get_fmt_names(filename_format)):
        for row, info in zip(rows, infos):
            row.update(get_url_names(info["url"]))

    # For the file name, we allow the _repindex special key.
    format_filename = partial(
        RepFormatter(colidx_to_name, missing_value).format,
        filename_format)
    subpaths = _format_filenames(format_filename, rows, infos)
    return infos, subpaths


def dlplugin(dataset=None, url_file=None, input_type="ext",
             url_format="{0}", filename_format="{1}",
             no_autometa=False, meta=None,
             message=None, dry_run=False, fast=False,
             ifexists=None, missing_value=None):
    """Create and update a dataset from a list of URLs.

    Parameters
    ----------
    dataset : Dataset
        Add the URLs to this dataset (or possibly subdatasets of this
        dataset).  An empty or non-existent directory is passed to
        create a new dataset.  New subdatasets can be specified with
        `filename_format`.
    url_file : str
        A file that contains URLs or information that can be used to
        construct URLs.  Depending on the value of `input_type`, this
        should be a CSV file (with a header as the first row) or a
        JSON file (structured as a list of objects with string
        values).
    input_type : {"ext", "csv", "json"}, optional
        Whether `url_file` should be considered a CSV file or a JSON
        file.  The default value, "ext", means to consider `url_file`
        as a JSON file if it ends with ".json".  Otherwise, treat it
        as a CSV file.
    url_format : str, optional
        A format string that specifies the URL for each entry.  This
        value is similar to a normal Python format string where the
        names from `url_file` (column names for a CSV or properties
        for JSON) are available as placeholders.  If `url_file` is a
        CSV file, a positional index can also be used (i.e., "{0}" for
        the first column).  Note that a placeholder cannot contain a
        ':' or '!'.
    filename_format : str, optional
        Like `url_format`, but this format string specifies the file to
        which the URL's content will be downloaded.  The file name may
        contain directories.  The separator "//" can be used to indicate
        that the left-side directory should be created as a new
        subdataset.

        In addition to the placeholders described in `url_format`, there
        are a few special placeholder.

          - "_repindex"

            The constructed file names must be unique across all fields
            rows.  To avoid collisions, the special placeholder
            "_repindex" can added to the formatter.  It value will start
            at 0 and increment every time a file name repeats.

          - "_urlN"

            Each part of the formatted URL is available.  For example,
            in "http://datalad.org/for/git-users", "_url0" and "_url1"
            would map to "for" and "git-users", respectively.

            The final part is also available as "_url_basename".
    no_autometa : bool, optional
        By default, metadata field=value pairs are constructed with each
        column in `url_file`, excluding any single column that is
        specified via `url_format`.  Set this flag to True to disable
        this behavior.  Metadata can still be set explicitly with the
        `meta` argument.
    meta : str, optional
        A format string that specifies metadata.  It should be
        structured as "<field>=<value>".  The same placeholders from
        `url_format` can be used.  As an example, "location={3}" would
        mean that the value for the "location" metadata field should be
        set the value of the fourth column.  This option can be given
        multiple times.
    message : str, optional
        Use this message when committing the URL additions.
    dry_run : bool, optional
        Report which URLs would be downloaded to which files and then
        exit.
    fast : bool, optional
        If True, add the URLs, but don't download their content.
        Underneath, this passes the --fast flag to `git annex addurl`.
    ifexists : {None, 'overwrite', 'skip'}
        What to do if a constructed file name already exists.  The
        default (None) behavior to proceed with the `git annex addurl`,
        which will failed if the file size has changed.  If set to
        'overwrite', remove the old file before adding the new one.  If
        set to 'skip', do not add the new file.
    missing_value : str, optional
        When an empty string is encountered, use this value instead.

    Examples
    --------
    Consider a file "avatars.csv" that contains

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and
    'ext' fields, we could run

        $ datalad plugin -d avatar_ds addurls url_file=avatars.csv
          url_format='{link}' filename_format='{who}.{ext}' fast=True

    The '-d avatar_ds' is used to create a new dataset in
    "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new
    subdataset in an "avatars" subdirectory, we could use "//" in the
    `filename_format` argument:

        $ datalad plugin addurls url_file=avatars.csv
          url_format='{link}' filename_format='avatars//{who}.{ext}'
          fast=True

    Note
    ----
    For users familiar with 'git annex addurl': A large part of this
    plugin's functionality can be viewed as transforming data from
    `url_file` into a "url filename" format that fed to 'git annex
    addurl --batch --with-files'.
    """
    import logging
    import os

    from datalad.distribution.dataset import Dataset
    from datalad.interface.results import get_status_dict
    import datalad.plugin.addurls as me
    from datalad.support.annexrepo import AnnexRepo
    from datalad.support.exceptions import AnnexBatchCommandError
    from datalad.utils import assure_list
    from datalad.ui import ui

    lgr = logging.getLogger("datalad.plugin.addurls")

    meta = assure_list(meta)

    if url_file is None:
        # `url_file` is not a required argument in `dlplugin` because
        # the argument before it, `dataset`, needs to be optional to
        # support the creation of new datasets.
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="Must specify url_file argument")
        return

    if dataset.repo and not isinstance(dataset.repo, AnnexRepo):
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message="not an annex repo")
        return

    if input_type == "ext":
        extension = os.path.splitext(url_file)[1]
        input_type = "json" if extension == ".json" else "csv"

    with open(url_file) as fd:
        rows, subpaths = me.extract(fd, input_type,
                                    filename_format, url_format,
                                    no_autometa, meta,
                                    missing_value)

    all_files = [row["filename"] for row in rows]
    if len(all_files) != len(set(all_files)):
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="error",
                              message=("There are file name collisions; "
                                       "consider using {_repindex}"))
        return

    if dry_run:
        for subpath in subpaths:
            lgr.info("Would create a subdataset at %s", subpath)
        for row in rows:
            lgr.info("Would download %s to %s",
                     row["url"], os.path.join(dataset.path, row["filename"]))
            lgr.info("Metadata: %s", row["meta_args"])
        yield get_status_dict(action="addurls",
                              ds=dataset,
                              status="ok",
                              message="dry-run finished")
        return

    if not dataset.repo:
        # Populate a new dataset with the URLs.
        for r in dataset.create(result_xfm=None, return_type='generator'):
            yield r

    annex_options = ["--fast"] if fast else []

    for spath in subpaths:
        if os.path.exists(os.path.join(dataset.path, spath)):
            lgr.warning(
                "Not creating subdataset at existing path: %s",
                spath)
        else:
            for r in dataset.create(spath, result_xfm=None,
                                    return_type='generator'):
                yield r

    pbar_addurl = ui.get_progressbar(total=len(rows), label="Adding files",
                                     unit=" Files")
    pbar_addurl.start()

    files_to_add = []
    meta_to_add = []
    addurl_results = []
    for row_idx, row in enumerate(rows, 1):
        pbar_addurl.update(row_idx)
        fname_abs = os.path.join(dataset.path, row["filename"])
        if row["subpath"]:
            # Adjust the dataset and filename for an `addurl` call
            # from within the subdataset that will actually contain
            # the link.
            ds_current = Dataset(os.path.join(dataset.path, row["subpath"]))
            ds_filename = os.path.relpath(fname_abs, ds_current.path)
        else:
            ds_current = dataset
            ds_filename = row["filename"]

        if os.path.exists(fname_abs) or os.path.islink(fname_abs):
            if ifexists == "skip":
                addurl_results.append(
                    get_status_dict(action="addurls",
                                    ds=ds_current,
                                    type="file",
                                    path=fname_abs,
                                    status="notneeded"))
                continue
            elif ifexists == "overwrite":
                lgr.debug("Removing %s", fname_abs)
                os.unlink(fname_abs)
            else:
                lgr.debug("File %s already exists", fname_abs)

        try:
            ds_current.repo.add_url_to_file(ds_filename, row["url"],
                                            batch=True, options=annex_options)
        except AnnexBatchCommandError:
            addurl_results.append(
                get_status_dict(action="addurls",
                                ds=ds_current,
                                type="file",
                                path=os.path.join(ds_current.path,
                                                  ds_filename),
                                message="failed to add URL",
                                status="error"))

            continue
        # Collect the status dicts to yield later so that we don't
        # interrupt the progress bar.
        addurl_results.append(
            get_status_dict(action="addurls",
                            ds=ds_current,
                            type="file",
                            path=os.path.join(ds_current.path,
                                              ds_filename),
                            status="ok"))

        files_to_add.append(row["filename"])
        meta_to_add.append((ds_current, ds_filename, row["meta_args"]))
    pbar_addurl.finish()

    for result in addurl_results:
        yield result

        msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)

    if files_to_add:
        for r in dataset.add(files_to_add, message=msg):
            yield r

    # TODO: Wrap this repeated "delayed results/progress bar" pattern.
    pbar_meta = ui.get_progressbar(total=len(meta_to_add),
                                   label="Adding metadata", unit=" Files")
    pbar_meta.start()

    meta_results = []
    for meta_idx, (ds, fname, meta) in enumerate(meta_to_add, 1):
        pbar_meta.update(meta_idx)
        lgr.debug("Adding metadata to %s in %s", fname, ds.path)
        for arg in meta:
            ds.repo._run_annex_command("metadata",
                                       annex_options=["--set", arg, fname])
        meta_results.append(
            get_status_dict(action="addurls-metadata",
                            ds=ds_current,
                            type="file",
                            path=os.path.join(ds.path, fname),
                            message="added metadata",
                            status="ok"))

    for result in meta_results:
        yield result

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user