{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport re\nimport string\n\nfrom six import string_types\nfrom six.moves.urllib.parse import urlparse\n\nfrom datalad.dochelpers import exc_str\nfrom datalad.log import log_progress, with_result_progress\nfrom datalad.interface.base import Interface\nfrom datalad.interface.base import build_doc\nfrom datalad.interface.results import annexjson2result, get_status_dict\nfrom datalad.interface.common_opts import nosave_opt\nfrom datalad.support.exceptions import AnnexBatchCommandError\nfrom datalad.support.network import get_url_filename\nfrom datalad.support.path import split_ext\nfrom datalad.support.s3 import get_versioned_url\nfrom datalad.utils import (\n    assure_list,\n    unlink,\n)\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name=None, missing_value=None):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, string_types):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Process metadata arguments.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    A dict mapping field names to values.\n    \"\"\"\n    results = {}\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n        results[field] = value\n    return results\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> from datalad.plugin.addurls import get_subpaths\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(*(spaths + [part]))\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    return bool(re.match(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\", name))\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n\n    Note: This is like `filter(is_legal_metafield, fields)` but the\n    dropped values are logged.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\ndef _read(stream, input_type):\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n    return rows, idx_map\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        filename = format_fn(row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_file_parts(filename, prefix=\"name\"):\n    \"\"\"Assign a name to various parts of a file.\n\n    Parameters\n    ----------\n    filename : str\n        A file name (no leading path is permitted).\n    prefix : str\n        Prefix to prepend to the key names.\n\n    Returns\n    -------\n    A dict mapping each part to a value.\n    \"\"\"\n    root, ext = split_ext(filename)\n    root_py, ext_py = os.path.splitext(filename)\n\n    return {prefix: filename,\n            prefix + \"_root\": root,\n            prefix + \"_ext\": ext,\n            prefix + \"_root_py\": root_py,\n            prefix + \"_ext_py\": ext_py}\n\n\ndef get_url_parts(url):\n    \"\"\"Assign a name to various parts of the URL.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys `_url_hostname` and, for a path with N+1 parts,\n    '_url0' through '_urlN' .  There is also a `_url_basename` key for\n    the rightmost part of the path.\n    \"\"\"\n    parsed = urlparse(url)\n    if not parsed.netloc:\n        return {}\n\n    names = {\"_url_hostname\": parsed.netloc}\n\n    path = parsed.path.strip(\"/\")\n    if not path:\n        return names\n\n    url_parts = path.split(\"/\")\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    basename = url_parts[-1]\n    names[\"_url_basename\"] = basename\n    names.update(get_file_parts(basename, prefix=\"_url_basename\"))\n    return names\n\n\ndef add_extra_filename_values(filename_format, rows, urls, dry_run):\n    \"\"\"Extend `rows` with values for special formatting fields.\n    \"\"\"\n    file_fields = list(get_fmt_names(filename_format))\n    if any(i.startswith(\"_url\") for i in file_fields):\n        for row, url in zip(rows, urls):\n            row.update(get_url_parts(url))\n\n    if any(i.startswith(\"_url_filename\") for i in file_fields):\n        if dry_run:  # Don't waste time making requests.\n            dummy = get_file_parts(\"BASE.EXT\", \"_url_filename\")\n            for idx, row in enumerate(rows):\n                row.update(\n                    {k: v + str(idx) for k, v in dummy.items()})\n        else:\n            num_urls = len(urls)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Requesting file names for %d URLs\", num_urls,\n                         label=\"Requesting names\", total=num_urls,\n                         unit=\" Files\")\n            for row, url in zip(rows, urls):\n                # If we run into any issues here, we're just going to raise an\n                # exception and then abort inside dlplugin.  It'd be good to\n                # disentangle this from `extract` so that we could yield an\n                # individual error, drop the row, and keep going.\n                filename = get_url_filename(url)\n                if filename:\n                    row.update(get_file_parts(filename, \"_url_filename\"))\n                else:\n                    raise ValueError(\n                        \"{} does not contain a filename\".format(url))\n                log_progress(lgr.info, \"addurls_requestnames\",\n                             \"%s returned for %s\", url, filename,\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Finished requesting file names\")\n\n\ndef extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    meta = assure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        url = format_url(row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        meta_args = clean_meta_args(fmt(row) for fmt in formats_meta)\n        infos.append({\"url\": url, \"meta_args\": meta_args})\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, subpaths\n\n\n@with_result_progress(\"Adding URLs\")\ndef add_urls(rows, ifexists=None, options=None):\n    \"\"\"Call `git annex addurl` using information in `rows`.\n    \"\"\"\n    for row in rows:\n        filename_abs = row[\"filename_abs\"]\n        ds, filename = row[\"ds\"], row[\"ds_filename\"]\n        lgr.debug(\"Adding metadata to %s in %s\", filename, ds.path)\n\n        if os.path.exists(filename_abs) or os.path.islink(filename_abs):\n            if ifexists == \"skip\":\n                yield get_status_dict(action=\"addurls\",\n                                      ds=ds,\n                                      type=\"file\",\n                                      path=filename_abs,\n                                      status=\"notneeded\")\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", filename_abs)\n                unlink(filename_abs)\n            else:\n                lgr.debug(\"File %s already exists\", filename_abs)\n\n        try:\n            out_json = ds.repo.add_url_to_file(filename, row[\"url\"],\n                                               batch=True, options=options)\n        except AnnexBatchCommandError as exc:\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds,\n                                  type=\"file\",\n                                  path=filename_abs,\n                                  message=exc_str(exc),\n                                  status=\"error\")\n            continue\n\n        # In the case of an error, the json object has file=None.\n        if out_json[\"file\"] is None:\n            out_json[\"file\"] = filename_abs\n        yield annexjson2result(out_json, ds, action=\"addurls\",\n                               type=\"file\", logger=lgr)\n\n\n@with_result_progress(\"Adding metadata\")\ndef add_meta(rows):\n    \"\"\"Call `git annex metadata --set` using information in `rows`.\n    \"\"\"\n    from mock import patch\n\n    for row in rows:\n        ds, filename = row[\"ds\"], row[\"ds_filename\"]\n\n        with patch.object(ds.repo, \"always_commit\", False):\n            lgr.debug(\"Adding metadata to %s in %s\", filename, ds.path)\n            for a in ds.repo.set_metadata(filename, add=row[\"meta_args\"]):\n                res = annexjson2result(a, ds, type=\"file\", logger=lgr)\n                # Don't show all added metadata for the file because that\n                # could quickly flood the output.\n                del res[\"message\"]\n                yield res\n\n\n@build_doc\nclass Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a CSV or\n    properties for JSON) are available as placeholders.  If `URL-FILE` is a CSV\n    file, a positional index can also be used (i.e., \"{0}\" for the first\n    column).  Note that a placeholder cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a CSV file (with a header as the first row) or a JSON\n            file (structured as a list of objects with string values).\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded.  The file name may\n            contain directories.  The separator \"//\" can be used to indicate\n            that the left-side directory should be created as a new subdataset.\n            See the 'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file or a JSON\n            file.  The default value, \"ext\", means to consider `URL-FILE` as a\n            JSON file if it ends with \".json\".  Otherwise, treat it as a CSV\n            file.\"\"\",\n            constraints=EnsureChoice(\"ext\", \"csv\", \"json\")),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            metavar=\"ACTION\",\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureNone() | EnsureChoice(\"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on URLs for AWS S3 buckets.\"\"\"),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.interface.results import get_status_dict\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        dataset = require_dataset(dataset, check_installed=False)\n        if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        if input_type == \"ext\":\n            extension = os.path.splitext(url_file)[1]\n            input_type = \"json\" if extension == \".json\" else \"csv\"\n\n        with open(url_file) as fd:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield get_status_dict(action=\"addurls\",\n                                      ds=dataset,\n                                      status=\"error\",\n                                      message=exc_str(exc))\n                return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=(\"There are file name collisions; \"\n                                           \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would download %s to %s\",\n                         row[\"url\"],\n                         os.path.join(dataset.path, row[\"filename\"]))\n                lgr.info(\"Metadata: %s\",\n                         sorted(u\"{}={}\".format(k, v)\n                                for k, v in row[\"meta_args\"].items()))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            for r in dataset.create(result_xfm=None, return_type='generator',\n                                    save=save):\n                yield r\n\n        annex_options = [\"--fast\"] if fast else []\n\n        for spath in subpaths:\n            if os.path.exists(os.path.join(dataset.path, spath)):\n                lgr.warning(\n                    \"Not creating subdataset at existing path: %s\",\n                    spath)\n            else:\n                for r in dataset.create(spath, result_xfm=None,\n                                        return_type='generator', save=save):\n                    yield r\n\n        for row in rows:\n            # Add additional information that we'll need for various\n            # operations.\n            filename_abs = os.path.join(dataset.path, row[\"filename\"])\n            if row[\"subpath\"]:\n                ds_current = Dataset(os.path.join(dataset.path,\n                                                  row[\"subpath\"]))\n                ds_filename = os.path.relpath(filename_abs, ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = row[\"filename\"]\n            row.update({\"filename_abs\": filename_abs,\n                        \"ds\": ds_current,\n                        \"ds_filename\": ds_filename})\n\n        if version_urls:\n            num_urls = len(rows)\n            log_progress(lgr.info, \"addurls_versionurls\",\n                         \"Versioning %d URLs\", num_urls,\n                         label=\"Versioning URLs\",\n                         total=num_urls, unit=\" URLs\")\n            for row in rows:\n                url = row[\"url\"]\n                try:\n                    row[\"url\"] = get_versioned_url(url)\n                except (ValueError, NotImplementedError) as exc:\n                    # We don't expect this to happen because get_versioned_url\n                    # should return the original URL if it isn't an S3 bucket.\n                    # It only raises exceptions if it doesn't know how to\n                    # handle the scheme for what looks like an S3 bucket.\n                    lgr.warning(\"error getting version of %s: %s\",\n                                row[\"url\"], exc_str(exc))\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioned result for %s: %s\", url, row[\"url\"],\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n        files_to_add = set()\n        for r in add_urls(rows, ifexists=ifexists, options=annex_options):\n            if r[\"status\"] == \"ok\":\n                files_to_add.add(r[\"path\"])\n            yield r\n\n            msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n        if files_to_add:\n            for r in dataset.add(files_to_add, save=False):\n                yield r\n\n            meta_rows = [r for r in rows if r[\"filename_abs\"] in files_to_add]\n            for r in add_meta(meta_rows):\n                yield r\n\n            # Save here rather than the add call above to trigger a metadata\n            # commit on the git-annex branch.\n            if save:\n                for r in dataset.save(message=msg, recursive=True):\n                    yield r\n\n\n__datalad_plugin__ = Addurls\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport re\nimport string\n\nfrom six import string_types\nfrom six.moves.urllib.parse import urlparse\n\nfrom datalad.dochelpers import exc_str\nfrom datalad.log import log_progress, with_result_progress\nfrom datalad.interface.base import Interface\nfrom datalad.interface.base import build_doc\nfrom datalad.interface.results import annexjson2result, get_status_dict\nfrom datalad.interface.common_opts import nosave_opt\nfrom datalad.support.exceptions import AnnexBatchCommandError\nfrom datalad.support.network import get_url_filename\nfrom datalad.support.path import split_ext\nfrom datalad.support.s3 import get_versioned_url\nfrom datalad.utils import (\n    assure_list,\n    unlink,\n)\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name=None, missing_value=None):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, string_types):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Process metadata arguments.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    A dict mapping field names to values.\n    \"\"\"\n    results = {}\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n        results[field] = value\n    return results\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> from datalad.plugin.addurls import get_subpaths\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(spaths[-1], part) if spaths else part\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    return bool(re.match(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\", name))\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n\n    Note: This is like `filter(is_legal_metafield, fields)` but the\n    dropped values are logged.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\ndef _read(stream, input_type):\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n    return rows, idx_map\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        filename = format_fn(row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_file_parts(filename, prefix=\"name\"):\n    \"\"\"Assign a name to various parts of a file.\n\n    Parameters\n    ----------\n    filename : str\n        A file name (no leading path is permitted).\n    prefix : str\n        Prefix to prepend to the key names.\n\n    Returns\n    -------\n    A dict mapping each part to a value.\n    \"\"\"\n    root, ext = split_ext(filename)\n    root_py, ext_py = os.path.splitext(filename)\n\n    return {prefix: filename,\n            prefix + \"_root\": root,\n            prefix + \"_ext\": ext,\n            prefix + \"_root_py\": root_py,\n            prefix + \"_ext_py\": ext_py}\n\n\ndef get_url_parts(url):\n    \"\"\"Assign a name to various parts of the URL.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys `_url_hostname` and, for a path with N+1 parts,\n    '_url0' through '_urlN' .  There is also a `_url_basename` key for\n    the rightmost part of the path.\n    \"\"\"\n    parsed = urlparse(url)\n    if not parsed.netloc:\n        return {}\n\n    names = {\"_url_hostname\": parsed.netloc}\n\n    path = parsed.path.strip(\"/\")\n    if not path:\n        return names\n\n    url_parts = path.split(\"/\")\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    basename = url_parts[-1]\n    names[\"_url_basename\"] = basename\n    names.update(get_file_parts(basename, prefix=\"_url_basename\"))\n    return names\n\n\ndef add_extra_filename_values(filename_format, rows, urls, dry_run):\n    \"\"\"Extend `rows` with values for special formatting fields.\n    \"\"\"\n    file_fields = list(get_fmt_names(filename_format))\n    if any(i.startswith(\"_url\") for i in file_fields):\n        for row, url in zip(rows, urls):\n            row.update(get_url_parts(url))\n\n    if any(i.startswith(\"_url_filename\") for i in file_fields):\n        if dry_run:  # Don't waste time making requests.\n            dummy = get_file_parts(\"BASE.EXT\", \"_url_filename\")\n            for idx, row in enumerate(rows):\n                row.update(\n                    {k: v + str(idx) for k, v in dummy.items()})\n        else:\n            num_urls = len(urls)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Requesting file names for %d URLs\", num_urls,\n                         label=\"Requesting names\", total=num_urls,\n                         unit=\" Files\")\n            for row, url in zip(rows, urls):\n                # If we run into any issues here, we're just going to raise an\n                # exception and then abort inside dlplugin.  It'd be good to\n                # disentangle this from `extract` so that we could yield an\n                # individual error, drop the row, and keep going.\n                filename = get_url_filename(url)\n                if filename:\n                    row.update(get_file_parts(filename, \"_url_filename\"))\n                else:\n                    raise ValueError(\n                        \"{} does not contain a filename\".format(url))\n                log_progress(lgr.info, \"addurls_requestnames\",\n                             \"%s returned for %s\", url, filename,\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Finished requesting file names\")\n\n\ndef sort_paths(paths):\n    \"\"\"Sort `paths` by directory level and then alphabetically.\n\n    Parameters\n    ----------\n    paths : iterable of str\n\n    Returns\n    -------\n    Generator of sorted paths.\n    \"\"\"\n    def level_and_name(p):\n        return p.count(os.path.sep), p\n\n    for path in sorted(paths, key=level_and_name):\n        yield path\n\n\ndef extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    meta = assure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        url = format_url(row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        meta_args = clean_meta_args(fmt(row) for fmt in formats_meta)\n        infos.append({\"url\": url, \"meta_args\": meta_args})\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, list(sort_paths(subpaths))\n\n\n@with_result_progress(\"Adding URLs\")\ndef add_urls(rows, ifexists=None, options=None):\n    \"\"\"Call `git annex addurl` using information in `rows`.\n    \"\"\"\n    for row in rows:\n        filename_abs = row[\"filename_abs\"]\n        ds, filename = row[\"ds\"], row[\"ds_filename\"]\n        lgr.debug(\"Adding metadata to %s in %s\", filename, ds.path)\n\n        if os.path.exists(filename_abs) or os.path.islink(filename_abs):\n            if ifexists == \"skip\":\n                yield get_status_dict(action=\"addurls\",\n                                      ds=ds,\n                                      type=\"file\",\n                                      path=filename_abs,\n                                      status=\"notneeded\")\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", filename_abs)\n                unlink(filename_abs)\n            else:\n                lgr.debug(\"File %s already exists\", filename_abs)\n\n        try:\n            out_json = ds.repo.add_url_to_file(filename, row[\"url\"],\n                                               batch=True, options=options)\n        except AnnexBatchCommandError as exc:\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds,\n                                  type=\"file\",\n                                  path=filename_abs,\n                                  message=exc_str(exc),\n                                  status=\"error\")\n            continue\n\n        # In the case of an error, the json object has file=None.\n        if out_json[\"file\"] is None:\n            out_json[\"file\"] = filename_abs\n        yield annexjson2result(out_json, ds, action=\"addurls\",\n                               type=\"file\", logger=lgr)\n\n\n@with_result_progress(\"Adding metadata\")\ndef add_meta(rows):\n    \"\"\"Call `git annex metadata --set` using information in `rows`.\n    \"\"\"\n    from mock import patch\n\n    for row in rows:\n        ds, filename = row[\"ds\"], row[\"ds_filename\"]\n\n        with patch.object(ds.repo, \"always_commit\", False):\n            lgr.debug(\"Adding metadata to %s in %s\", filename, ds.path)\n            for a in ds.repo.set_metadata(filename, add=row[\"meta_args\"]):\n                res = annexjson2result(a, ds, type=\"file\", logger=lgr)\n                # Don't show all added metadata for the file because that\n                # could quickly flood the output.\n                del res[\"message\"]\n                yield res\n\n\n@build_doc\nclass Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a CSV or\n    properties for JSON) are available as placeholders.  If `URL-FILE` is a CSV\n    file, a positional index can also be used (i.e., \"{0}\" for the first\n    column).  Note that a placeholder cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a CSV file (with a header as the first row) or a JSON\n            file (structured as a list of objects with string values).\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded.  The file name may\n            contain directories.  The separator \"//\" can be used to indicate\n            that the left-side directory should be created as a new subdataset.\n            See the 'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file or a JSON\n            file.  The default value, \"ext\", means to consider `URL-FILE` as a\n            JSON file if it ends with \".json\".  Otherwise, treat it as a CSV\n            file.\"\"\",\n            constraints=EnsureChoice(\"ext\", \"csv\", \"json\")),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            metavar=\"ACTION\",\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureNone() | EnsureChoice(\"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on URLs for AWS S3 buckets.\"\"\"),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.interface.results import get_status_dict\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        dataset = require_dataset(dataset, check_installed=False)\n        if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        if input_type == \"ext\":\n            extension = os.path.splitext(url_file)[1]\n            input_type = \"json\" if extension == \".json\" else \"csv\"\n\n        with open(url_file) as fd:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield get_status_dict(action=\"addurls\",\n                                      ds=dataset,\n                                      status=\"error\",\n                                      message=exc_str(exc))\n                return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=(\"There are file name collisions; \"\n                                           \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would download %s to %s\",\n                         row[\"url\"],\n                         os.path.join(dataset.path, row[\"filename\"]))\n                lgr.info(\"Metadata: %s\",\n                         sorted(u\"{}={}\".format(k, v)\n                                for k, v in row[\"meta_args\"].items()))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            for r in dataset.create(result_xfm=None, return_type='generator',\n                                    save=save):\n                yield r\n\n        annex_options = [\"--fast\"] if fast else []\n\n        for spath in subpaths:\n            if os.path.exists(os.path.join(dataset.path, spath)):\n                lgr.warning(\n                    \"Not creating subdataset at existing path: %s\",\n                    spath)\n            else:\n                for r in dataset.create(spath, result_xfm=None,\n                                        return_type='generator', save=save):\n                    yield r\n\n        for row in rows:\n            # Add additional information that we'll need for various\n            # operations.\n            filename_abs = os.path.join(dataset.path, row[\"filename\"])\n            if row[\"subpath\"]:\n                ds_current = Dataset(os.path.join(dataset.path,\n                                                  row[\"subpath\"]))\n                ds_filename = os.path.relpath(filename_abs, ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = row[\"filename\"]\n            row.update({\"filename_abs\": filename_abs,\n                        \"ds\": ds_current,\n                        \"ds_filename\": ds_filename})\n\n        if version_urls:\n            num_urls = len(rows)\n            log_progress(lgr.info, \"addurls_versionurls\",\n                         \"Versioning %d URLs\", num_urls,\n                         label=\"Versioning URLs\",\n                         total=num_urls, unit=\" URLs\")\n            for row in rows:\n                url = row[\"url\"]\n                try:\n                    row[\"url\"] = get_versioned_url(url)\n                except (ValueError, NotImplementedError) as exc:\n                    # We don't expect this to happen because get_versioned_url\n                    # should return the original URL if it isn't an S3 bucket.\n                    # It only raises exceptions if it doesn't know how to\n                    # handle the scheme for what looks like an S3 bucket.\n                    lgr.warning(\"error getting version of %s: %s\",\n                                row[\"url\"], exc_str(exc))\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioned result for %s: %s\", url, row[\"url\"],\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n        files_to_add = set()\n        for r in add_urls(rows, ifexists=ifexists, options=annex_options):\n            if r[\"status\"] == \"ok\":\n                files_to_add.add(r[\"path\"])\n            yield r\n\n            msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n        if files_to_add:\n            for r in dataset.add(files_to_add, save=False):\n                yield r\n\n            meta_rows = [r for r in rows if r[\"filename_abs\"] in files_to_add]\n            for r in add_meta(meta_rows):\n                yield r\n\n            # Save here rather than the add call above to trigger a metadata\n            # commit on the git-annex branch.\n            if save:\n                for r in dataset.save(message=msg, recursive=True):\n                    yield r\n\n\n__datalad_plugin__ = Addurls\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "23100e40c",
                "new_commit_hash": "a98ebc00a",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 176,
                            "length": 7
                        },
                        "new_range": {
                            "start": 176,
                            "length": 7
                        },
                        "section": "def get_subpaths(filename):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    spaths = []",
                                "type": "context"
                            },
                            {
                                "content": "    for part in filename.split(\"//\")[:-1]:",
                                "type": "context"
                            },
                            {
                                "content": "        path = os.path.join(*(spaths + [part]))",
                                "type": "deleted"
                            },
                            {
                                "content": "        path = os.path.join(spaths[-1], part) if spaths else part",
                                "type": "added"
                            },
                            {
                                "content": "        spaths.append(path)",
                                "type": "context"
                            },
                            {
                                "content": "    return filename.replace(\"//\", os.path.sep), spaths",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "get_subpaths",
                            "content": "def get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> from datalad.plugin.addurls import get_subpaths\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(*(spaths + [part]))\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 153,
                            "end_lineno": 181,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 373,
                            "length": 6
                        },
                        "new_range": {
                            "start": 373,
                            "length": 24
                        },
                        "section": "def add_extra_filename_values(filename_format, rows, urls, dry_run):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                         \"Finished requesting file names\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def sort_paths(paths):",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Sort `paths` by directory level and then alphabetically.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Parameters",
                                "type": "added"
                            },
                            {
                                "content": "    ----------",
                                "type": "added"
                            },
                            {
                                "content": "    paths : iterable of str",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Returns",
                                "type": "added"
                            },
                            {
                                "content": "    -------",
                                "type": "added"
                            },
                            {
                                "content": "    Generator of sorted paths.",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "    def level_and_name(p):",
                                "type": "added"
                            },
                            {
                                "content": "        return p.count(os.path.sep), p",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    for path in sorted(paths, key=level_and_name):",
                                "type": "added"
                            },
                            {
                                "content": "        yield path",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",",
                                "type": "context"
                            },
                            {
                                "content": "            exclude_autometa=None, meta=None,",
                                "type": "context"
                            },
                            {
                                "content": "            dry_run=False, missing_value=None):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "sort_paths",
                            "content": "def sort_paths(paths):\n    \"\"\"Sort `paths` by directory level and then alphabetically.\n\n    Parameters\n    ----------\n    paths : iterable of str\n\n    Returns\n    -------\n    Generator of sorted paths.\n    \"\"\"\n    def level_and_name(p):\n        return p.count(os.path.sep), p\n\n    for path in sorted(paths, key=level_and_name):\n        yield path",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 376,
                            "end_lineno": 391,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 440,
                            "length": 7
                        },
                        "new_range": {
                            "start": 458,
                            "length": 7
                        },
                        "section": "def extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        RepFormatter(colidx_to_name, missing_value).format,",
                                "type": "context"
                            },
                            {
                                "content": "        filename_format)",
                                "type": "context"
                            },
                            {
                                "content": "    subpaths = _format_filenames(format_filename, rows_with_url, infos)",
                                "type": "context"
                            },
                            {
                                "content": "    return infos, subpaths",
                                "type": "deleted"
                            },
                            {
                                "content": "    return infos, list(sort_paths(subpaths))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@with_result_progress(\"Adding URLs\")",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    meta = assure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        url = format_url(row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        meta_args = clean_meta_args(fmt(row) for fmt in formats_meta)\n        infos.append({\"url\": url, \"meta_args\": meta_args})\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, subpaths",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 376,
                            "end_lineno": 443,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport tempfile\n\nfrom mock import patch\n\nfrom six.moves import StringIO\n\nfrom datalad.api import addurls, Dataset, subdatasets\nimport datalad.plugin.addurls as au\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.tests.utils import chpwd, slow, swallow_logs\nfrom datalad.tests.utils import assert_false, assert_true, assert_raises\nfrom datalad.tests.utils import assert_in, assert_re_in, assert_in_results\nfrom datalad.tests.utils import assert_dict_equal\nfrom datalad.tests.utils import eq_, ok_exists\nfrom datalad.tests.utils import create_tree, with_tempfile, HTTPPath\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = au.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = au.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = au.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = au.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = au.Formatter({})\n    eq_(fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}), \"value0\")\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = au.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    eq_(fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\"), \"'x'\")\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    eq_(fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\"), \"x    \")\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = au.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = au.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], {}),\n                         ([\" field=yes \"], {\"field\": \"yes\"}),\n                         ([\"field= value=\"], {\"field\": \"value=\"})]:\n        eq_(au.clean_meta_args(args), expect)\n\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"noequal\"])\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"=value\"])\n\n\ndef test_get_subpaths():\n    for fname, expect in [(\"no/dbl/slash\", (\"no/dbl/slash\", [])),\n                          (\"p1//n\", (\"p1/n\", [\"p1\"])),\n                          (\"p1//p2/p3//n\", (\"p1/p2/p3/n\",\n                                            [\"p1\", \"p1/p2/p3\"])),\n                          (\"//n\", (\"/n\", [\"\"])),\n                          (\"n//\", (\"n/\", [\"n\"]))]:\n        eq_(au.get_subpaths(fname), expect)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(au.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(au.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(au.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(au.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(au.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(au.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(au.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(au.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(au.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(au.fmt_to_name(\"\", {}))\n    assert_false(au.fmt_to_name(\"nonames\", {}))\n    assert_false(au.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_file_parts():\n    assert_dict_equal(au.get_file_parts(\"file.tar.gz\", \"prefix\"),\n                      {\"prefix\": \"file.tar.gz\",\n                       \"prefix_root_py\": \"file.tar\",\n                       \"prefix_ext_py\": \".gz\",\n                       \"prefix_root\": \"file\",\n                       \"prefix_ext\": \".tar.gz\"})\n\n\ndef test_get_url_parts():\n    eq_(au.get_url_parts(\"\"), {})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org\"),\n                      {\"_url_hostname\": \"datalad.org\"})\n\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      {\"_url_hostname\": \"datalad.org\",\n                       \"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\",\n                       \"_url_basename_root_py\": \"about\",\n                       \"_url_basename_ext_py\": \".html\",\n                       \"_url_basename_root\": \"about\",\n                       \"_url_basename_ext\": \".html\"})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      au.get_url_parts(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        au.get_url_parts(\"http://datalad.org/for/git-users\"),\n        {\"_url_hostname\": \"datalad.org\",\n         \"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\",\n         \"_url_basename_root_py\": \"git-users\",\n         \"_url_basename_ext_py\": \"\",\n         \"_url_basename_root\": \"git-users\",\n         \"_url_basename_ext\": \"\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])\n\n\ndef test_extract_disable_autometa():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"*\",\n        meta=[\"group={age_group}\"])\n\n    eq_([d[\"meta_args\"] for d in info],\n        [{\"group\": \"kid\"}, {\"group\": \"adult\"}, {\"group\": \"adult\"},\n         {\"group\": \"kid\"}])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"ea\")\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\"},\n               {\"name\": \"max\", \"age_group\": \"kid\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    kwds = dict(filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n                url_format=\"{name}_{debut_season}.com\",\n                meta=[\"group={age_group}\"])\n\n    json_output = au.extract(json_stream(ST_DATA[\"rows\"]), \"json\", **kwds)\n    csv_output = au.extract(csv_rows, \"csv\", **kwds)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  au.extract, None, \"not_csv_or_json\")\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, no_annex=True)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.addurls(\"dummy_arg0\", \"dummy_arg1\", \"dummy_arg2\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    with chpwd(path):\n        json_file = \"links.json\"\n        with open(json_file, \"w\") as jfh:\n            json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                       {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                       {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                      jfh)\n\n        ds.add(\".\", message=\"setup\")\n\n        with swallow_logs(new_level=logging.INFO) as cml:\n            ds.addurls(json_file,\n                       \"{url}\",\n                       \"{subdir}//{_url_filename_root}\",\n                       dry_run=True)\n\n            for dir_ in [\"foo\", \"bar\"]:\n                assert_in(\"Would create a subdataset at {}\".format(dir_),\n                          cml.out)\n            assert_in(\n                \"Would download URL/a.dat to {}\".format(\n                    os.path.join(path, \"foo\", \"BASE\")),\n                cml.out)\n\n            assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                      cml.out)\n\n\n@slow  # ~9s\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return int(\n                ds.repo.repo.git.rev_list(\"--count\", \"git-annex\").strip())\n\n        n_annex_commits = get_annex_commit_counts()\n\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                             [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(meta,\n                                  {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Ignore this check if we're faking dates because that disables\n            # batch mode.\n            if not os.environ.get('DATALAD_FAKE__DATES'):\n                # We should have two new commits on the git-annex: one for the\n                # added urls and one for the added metadata.\n                eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            for save in True, False:\n                label = \"save\" if save else \"nosave\"\n                hexsha_before = ds.repo.get_hexsha()\n                ds.addurls(self.json_file, \"{url}\",\n                           \"{subdir}-\" + label + \"//{name}\",\n                           save=save)\n                hexsha_after = ds.repo.get_hexsha()\n\n                for fname in [\"foo-{}/a\", \"bar-{}/b\", \"foo-{}/c\"]:\n                    ok_exists(fname.format(label))\n\n                assert_true(save ^ (hexsha_before == hexsha_after))\n                assert_true(save ^ ds.repo.dirty)\n\n            # Now save the \"--nosave\" changes and check that we have\n            # all the subdatasets.\n            ds.add(\".\")\n            eq_(set(subdatasets(ds, recursive=True,\n                                result_xfm=\"relpaths\")),\n                {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file,\n                          \"{url}/nofilename/\",\n                          \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with chpwd(path), patch.object(ds.repo, 'set_metadata', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path), swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[ds.repo.WEB_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n",
            "new_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport os.path as op\nimport tempfile\n\nfrom mock import patch\n\nfrom six.moves import StringIO\n\nfrom datalad.api import addurls, Dataset, subdatasets\nimport datalad.plugin.addurls as au\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.tests.utils import chpwd, slow, swallow_logs\nfrom datalad.tests.utils import assert_false, assert_true, assert_raises\nfrom datalad.tests.utils import assert_in, assert_re_in, assert_in_results\nfrom datalad.tests.utils import assert_dict_equal\nfrom datalad.tests.utils import eq_, ok_exists\nfrom datalad.tests.utils import create_tree, with_tempfile, HTTPPath\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = au.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = au.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = au.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = au.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = au.Formatter({})\n    eq_(fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}), \"value0\")\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = au.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    eq_(fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\"), \"'x'\")\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    eq_(fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\"), \"x    \")\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = au.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = au.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], {}),\n                         ([\" field=yes \"], {\"field\": \"yes\"}),\n                         ([\"field= value=\"], {\"field\": \"value=\"})]:\n        eq_(au.clean_meta_args(args), expect)\n\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"noequal\"])\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"=value\"])\n\n\ndef test_get_subpaths():\n    for fname, expect in [\n            (op.join(\"no\", \"dbl\", \"slash\"),\n             (op.join(\"no\", \"dbl\", \"slash\"), [])),\n            (\"p1//n\",\n             (op.join(\"p1\", \"n\"), [\"p1\"])),\n            (op.join(\"p1//p2\", \"p3//n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"n\"),\n              [\"p1\", op.join(\"p1\", \"p2\", \"p3\")])),\n            (op.join(\"p1//p2\", \"p3//p4\", \"p5//\", \"n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\", \"n\"),\n              [\"p1\",\n               op.join(\"p1\", \"p2\", \"p3\"),\n               op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\")])),\n            (\"//n\", (op.sep + \"n\", [\"\"])),\n            (\"n//\", (\"n\" + op.sep, [\"n\"]))]:\n        eq_(au.get_subpaths(fname), expect)\n\n\ndef test_sort_paths():\n    paths = [op.join(\"x\", \"a\", \"b\"),\n             \"z\",\n             op.join(\"y\", \"b\"),\n             op.join(\"y\", \"a\")]\n    expected = [\"z\",\n                op.join(\"y\", \"a\"),\n                op.join(\"y\", \"b\"),\n                op.join(\"x\", \"a\", \"b\")]\n    eq_(list(au.sort_paths(paths)), expected)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(au.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(au.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(au.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(au.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(au.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(au.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(au.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(au.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(au.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(au.fmt_to_name(\"\", {}))\n    assert_false(au.fmt_to_name(\"nonames\", {}))\n    assert_false(au.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_file_parts():\n    assert_dict_equal(au.get_file_parts(\"file.tar.gz\", \"prefix\"),\n                      {\"prefix\": \"file.tar.gz\",\n                       \"prefix_root_py\": \"file.tar\",\n                       \"prefix_ext_py\": \".gz\",\n                       \"prefix_root\": \"file\",\n                       \"prefix_ext\": \".tar.gz\"})\n\n\ndef test_get_url_parts():\n    eq_(au.get_url_parts(\"\"), {})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org\"),\n                      {\"_url_hostname\": \"datalad.org\"})\n\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      {\"_url_hostname\": \"datalad.org\",\n                       \"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\",\n                       \"_url_basename_root_py\": \"about\",\n                       \"_url_basename_ext_py\": \".html\",\n                       \"_url_basename_root\": \"about\",\n                       \"_url_basename_ext\": \".html\"})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      au.get_url_parts(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        au.get_url_parts(\"http://datalad.org/for/git-users\"),\n        {\"_url_hostname\": \"datalad.org\",\n         \"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\",\n         \"_url_basename_root_py\": \"git-users\",\n         \"_url_basename_ext_py\": \"\",\n         \"_url_basename_root\": \"git-users\",\n         \"_url_basename_ext\": \"\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        [\"adult\", \"kid\", \"adult/no\", \"adult/yes\", \"kid/no\"])\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])\n\n\ndef test_extract_disable_autometa():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"*\",\n        meta=[\"group={age_group}\"])\n\n    eq_([d[\"meta_args\"] for d in info],\n        [{\"group\": \"kid\"}, {\"group\": \"adult\"}, {\"group\": \"adult\"},\n         {\"group\": \"kid\"}])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"ea\")\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\"},\n               {\"name\": \"max\", \"age_group\": \"kid\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    kwds = dict(filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n                url_format=\"{name}_{debut_season}.com\",\n                meta=[\"group={age_group}\"])\n\n    json_output = au.extract(json_stream(ST_DATA[\"rows\"]), \"json\", **kwds)\n    csv_output = au.extract(csv_rows, \"csv\", **kwds)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  au.extract, None, \"not_csv_or_json\")\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, no_annex=True)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.addurls(\"dummy_arg0\", \"dummy_arg1\", \"dummy_arg2\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    with chpwd(path):\n        json_file = \"links.json\"\n        with open(json_file, \"w\") as jfh:\n            json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                       {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                       {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                      jfh)\n\n        ds.add(\".\", message=\"setup\")\n\n        with swallow_logs(new_level=logging.INFO) as cml:\n            ds.addurls(json_file,\n                       \"{url}\",\n                       \"{subdir}//{_url_filename_root}\",\n                       dry_run=True)\n\n            for dir_ in [\"foo\", \"bar\"]:\n                assert_in(\"Would create a subdataset at {}\".format(dir_),\n                          cml.out)\n            assert_in(\n                \"Would download URL/a.dat to {}\".format(\n                    os.path.join(path, \"foo\", \"BASE\")),\n                cml.out)\n\n            assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                      cml.out)\n\n\n@slow  # ~9s\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return int(\n                ds.repo.repo.git.rev_list(\"--count\", \"git-annex\").strip())\n\n        n_annex_commits = get_annex_commit_counts()\n\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                             [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(meta,\n                                  {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Ignore this check if we're faking dates because that disables\n            # batch mode.\n            if not os.environ.get('DATALAD_FAKE__DATES'):\n                # We should have two new commits on the git-annex: one for the\n                # added urls and one for the added metadata.\n                eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            for save in True, False:\n                label = \"save\" if save else \"nosave\"\n                hexsha_before = ds.repo.get_hexsha()\n                ds.addurls(self.json_file, \"{url}\",\n                           \"{subdir}-\" + label + \"//{name}\",\n                           save=save)\n                hexsha_after = ds.repo.get_hexsha()\n\n                for fname in [\"foo-{}/a\", \"bar-{}/b\", \"foo-{}/c\"]:\n                    ok_exists(fname.format(label))\n\n                assert_true(save ^ (hexsha_before == hexsha_after))\n                assert_true(save ^ ds.repo.dirty)\n\n            # Now save the \"--nosave\" changes and check that we have\n            # all the subdatasets.\n            ds.add(\".\")\n            eq_(set(subdatasets(ds, recursive=True,\n                                result_xfm=\"relpaths\")),\n                {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file,\n                          \"{url}/nofilename/\",\n                          \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with chpwd(path), patch.object(ds.repo, 'set_metadata', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path), swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[ds.repo.WEB_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\")\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/tests/test_addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "a2e00f4c4",
                "new_commit_hash": "b4d2ef6ca",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/tests/test_addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/tests/test_addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 12,
                            "length": 6
                        },
                        "new_range": {
                            "start": 12,
                            "length": 7
                        },
                        "section": ""
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "import json",
                                "type": "context"
                            },
                            {
                                "content": "import logging",
                                "type": "context"
                            },
                            {
                                "content": "import os",
                                "type": "context"
                            },
                            {
                                "content": "import os.path as op",
                                "type": "added"
                            },
                            {
                                "content": "import tempfile",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from mock import patch",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "import",
                            "name": "os.path",
                            "content": "import os.path as op",
                            "ast_type_str": "Import",
                            "start_lineno": 15,
                            "end_lineno": 15,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 117,
                            "length": 15
                        },
                        "new_range": {
                            "start": 118,
                            "length": 36
                        },
                        "section": "def test_clean_meta_args():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_get_subpaths():",
                                "type": "context"
                            },
                            {
                                "content": "    for fname, expect in [(\"no/dbl/slash\", (\"no/dbl/slash\", [])),",
                                "type": "deleted"
                            },
                            {
                                "content": "                          (\"p1//n\", (\"p1/n\", [\"p1\"])),",
                                "type": "deleted"
                            },
                            {
                                "content": "                          (\"p1//p2/p3//n\", (\"p1/p2/p3/n\",",
                                "type": "deleted"
                            },
                            {
                                "content": "                                            [\"p1\", \"p1/p2/p3\"])),",
                                "type": "deleted"
                            },
                            {
                                "content": "                          (\"//n\", (\"/n\", [\"\"])),",
                                "type": "deleted"
                            },
                            {
                                "content": "                          (\"n//\", (\"n/\", [\"n\"]))]:",
                                "type": "deleted"
                            },
                            {
                                "content": "    for fname, expect in [",
                                "type": "added"
                            },
                            {
                                "content": "            (op.join(\"no\", \"dbl\", \"slash\"),",
                                "type": "added"
                            },
                            {
                                "content": "             (op.join(\"no\", \"dbl\", \"slash\"), [])),",
                                "type": "added"
                            },
                            {
                                "content": "            (\"p1//n\",",
                                "type": "added"
                            },
                            {
                                "content": "             (op.join(\"p1\", \"n\"), [\"p1\"])),",
                                "type": "added"
                            },
                            {
                                "content": "            (op.join(\"p1//p2\", \"p3//n\"),",
                                "type": "added"
                            },
                            {
                                "content": "             (op.join(\"p1\", \"p2\", \"p3\", \"n\"),",
                                "type": "added"
                            },
                            {
                                "content": "              [\"p1\", op.join(\"p1\", \"p2\", \"p3\")])),",
                                "type": "added"
                            },
                            {
                                "content": "            (op.join(\"p1//p2\", \"p3//p4\", \"p5//\", \"n\"),",
                                "type": "added"
                            },
                            {
                                "content": "             (op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\", \"n\"),",
                                "type": "added"
                            },
                            {
                                "content": "              [\"p1\",",
                                "type": "added"
                            },
                            {
                                "content": "               op.join(\"p1\", \"p2\", \"p3\"),",
                                "type": "added"
                            },
                            {
                                "content": "               op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\")])),",
                                "type": "added"
                            },
                            {
                                "content": "            (\"//n\", (op.sep + \"n\", [\"\"])),",
                                "type": "added"
                            },
                            {
                                "content": "            (\"n//\", (\"n\" + op.sep, [\"n\"]))]:",
                                "type": "added"
                            },
                            {
                                "content": "        eq_(au.get_subpaths(fname), expect)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_sort_paths():",
                                "type": "added"
                            },
                            {
                                "content": "    paths = [op.join(\"x\", \"a\", \"b\"),",
                                "type": "added"
                            },
                            {
                                "content": "             \"z\",",
                                "type": "added"
                            },
                            {
                                "content": "             op.join(\"y\", \"b\"),",
                                "type": "added"
                            },
                            {
                                "content": "             op.join(\"y\", \"a\")]",
                                "type": "added"
                            },
                            {
                                "content": "    expected = [\"z\",",
                                "type": "added"
                            },
                            {
                                "content": "                op.join(\"y\", \"a\"),",
                                "type": "added"
                            },
                            {
                                "content": "                op.join(\"y\", \"b\"),",
                                "type": "added"
                            },
                            {
                                "content": "                op.join(\"x\", \"a\", \"b\")]",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(list(au.sort_paths(paths)), expected)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def test_is_legal_metafield():",
                                "type": "context"
                            },
                            {
                                "content": "    for legal in [\"legal\", \"0\", \"legal_\"]:",
                                "type": "context"
                            },
                            {
                                "content": "        assert_true(au.is_legal_metafield(legal))",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_get_subpaths",
                            "content": "def test_get_subpaths():\n    for fname, expect in [(\"no/dbl/slash\", (\"no/dbl/slash\", [])),\n                          (\"p1//n\", (\"p1/n\", [\"p1\"])),\n                          (\"p1//p2/p3//n\", (\"p1/p2/p3/n\",\n                                            [\"p1\", \"p1/p2/p3\"])),\n                          (\"//n\", (\"/n\", [\"\"])),\n                          (\"n//\", (\"n/\", [\"n\"]))]:\n        eq_(au.get_subpaths(fname), expect)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 119,
                            "end_lineno": 126,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_sort_paths",
                            "content": "def test_sort_paths():\n    paths = [op.join(\"x\", \"a\", \"b\"),\n             \"z\",\n             op.join(\"y\", \"b\"),\n             op.join(\"y\", \"a\")]\n    expected = [\"z\",\n                op.join(\"y\", \"a\"),\n                op.join(\"y\", \"b\"),\n                op.join(\"x\", \"a\", \"b\")]\n    eq_(list(au.sort_paths(paths)), expected)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 139,
                            "end_lineno": 148,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 213,
                            "length": 7
                        },
                        "new_range": {
                            "start": 235,
                            "length": 7
                        },
                        "section": "def test_extract():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        filename_format=\"{age_group}//{now_dead}//{name}.csv\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(subpaths,",
                                "type": "context"
                            },
                            {
                                "content": "        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})",
                                "type": "deleted"
                            },
                            {
                                "content": "        [\"adult\", \"kid\", \"adult/no\", \"adult/yes\", \"kid/no\"])",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    eq_([d[\"url\"] for d in info],",
                                "type": "context"
                            },
                            {
                                "content": "        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract",
                            "content": "def test_extract():\n    info, subpaths = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        {\"kid\", \"kid/no\", \"adult\", \"adult/yes\", \"adult/no\"})\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 209,
                            "end_lineno": 237,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 536,
                            "length": 3
                        },
                        "new_range": {
                            "start": 558,
                            "length": 19
                        },
                        "section": "class TestAddurls(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        for fname, info in whereis.items():",
                                "type": "context"
                            },
                            {
                                "content": "            eq_(info[ds.repo.WEB_UUID]['urls'],",
                                "type": "context"
                            },
                            {
                                "content": "                [\"{}udir/{}.dat.v1\".format(self.url, fname)])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    @with_tempfile(mkdir=True)",
                                "type": "added"
                            },
                            {
                                "content": "    def test_addurls_deeper(self, path):",
                                "type": "added"
                            },
                            {
                                "content": "        ds = Dataset(path).create(force=True)",
                                "type": "added"
                            },
                            {
                                "content": "        ds.addurls(",
                                "type": "added"
                            },
                            {
                                "content": "            self.json_file, \"{url}\",",
                                "type": "added"
                            },
                            {
                                "content": "            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\")",
                                "type": "added"
                            },
                            {
                                "content": "        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),",
                                "type": "added"
                            },
                            {
                                "content": "            {\"foo\",",
                                "type": "added"
                            },
                            {
                                "content": "             \"bar\",",
                                "type": "added"
                            },
                            {
                                "content": "             op.join(\"foo\", \"adir\", \"foo-again\"),",
                                "type": "added"
                            },
                            {
                                "content": "             op.join(\"bar\", \"adir\", \"bar-again\"),",
                                "type": "added"
                            },
                            {
                                "content": "             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),",
                                "type": "added"
                            },
                            {
                                "content": "             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})",
                                "type": "added"
                            },
                            {
                                "content": "        ok_exists(os.path.join(",
                                "type": "added"
                            },
                            {
                                "content": "            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "class",
                            "name": "TestAddurls",
                            "content": "class TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return int(\n                ds.repo.repo.git.rev_list(\"--count\", \"git-annex\").strip())\n\n        n_annex_commits = get_annex_commit_counts()\n\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                             [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(meta,\n                                  {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Ignore this check if we're faking dates because that disables\n            # batch mode.\n            if not os.environ.get('DATALAD_FAKE__DATES'):\n                # We should have two new commits on the git-annex: one for the\n                # added urls and one for the added metadata.\n                eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            for save in True, False:\n                label = \"save\" if save else \"nosave\"\n                hexsha_before = ds.repo.get_hexsha()\n                ds.addurls(self.json_file, \"{url}\",\n                           \"{subdir}-\" + label + \"//{name}\",\n                           save=save)\n                hexsha_after = ds.repo.get_hexsha()\n\n                for fname in [\"foo-{}/a\", \"bar-{}/b\", \"foo-{}/c\"]:\n                    ok_exists(fname.format(label))\n\n                assert_true(save ^ (hexsha_before == hexsha_after))\n                assert_true(save ^ ds.repo.dirty)\n\n            # Now save the \"--nosave\" changes and check that we have\n            # all the subdatasets.\n            ds.add(\".\")\n            eq_(set(subdatasets(ds, recursive=True,\n                                result_xfm=\"relpaths\")),\n                {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file,\n                          \"{url}/nofilename/\",\n                          \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with chpwd(path), patch.object(ds.repo, 'set_metadata', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path), swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[ds.repo.WEB_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\")\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 352,
                            "end_lineno": 576,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "method",
                            "name": "TestAddurls.test_addurls_deeper",
                            "content": "class TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return int(\n                ds.repo.repo.git.rev_list(\"--count\", \"git-annex\").strip())\n\n        n_annex_commits = get_annex_commit_counts()\n\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                             [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(meta,\n                                  {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Ignore this check if we're faking dates because that disables\n            # batch mode.\n            if not os.environ.get('DATALAD_FAKE__DATES'):\n                # We should have two new commits on the git-annex: one for the\n                # added urls and one for the added metadata.\n                eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            for save in True, False:\n                label = \"save\" if save else \"nosave\"\n                hexsha_before = ds.repo.get_hexsha()\n                ds.addurls(self.json_file, \"{url}\",\n                           \"{subdir}-\" + label + \"//{name}\",\n                           save=save)\n                hexsha_after = ds.repo.get_hexsha()\n\n                for fname in [\"foo-{}/a\", \"bar-{}/b\", \"foo-{}/c\"]:\n                    ok_exists(fname.format(label))\n\n                assert_true(save ^ (hexsha_before == hexsha_after))\n                assert_true(save ^ ds.repo.dirty)\n\n            # Now save the \"--nosave\" changes and check that we have\n            # all the subdatasets.\n            ds.add(\".\")\n            eq_(set(subdatasets(ds, recursive=True,\n                                result_xfm=\"relpaths\")),\n                {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file,\n                          \"{url}/nofilename/\",\n                          \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with chpwd(path), patch.object(ds.repo, 'set_metadata', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path), swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[ds.repo.WEB_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\")\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 563,
                            "end_lineno": 576,
                            "parent": {
                                "file_name": "datalad/plugin/tests/test_addurls.py",
                                "type": "class",
                                "name": "TestAddurls",
                                "content": "class TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return int(\n                ds.repo.repo.git.rev_list(\"--count\", \"git-annex\").strip())\n\n        n_annex_commits = get_annex_commit_counts()\n\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                             [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(meta,\n                                  {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Ignore this check if we're faking dates because that disables\n            # batch mode.\n            if not os.environ.get('DATALAD_FAKE__DATES'):\n                # We should have two new commits on the git-annex: one for the\n                # added urls and one for the added metadata.\n                eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            for save in True, False:\n                label = \"save\" if save else \"nosave\"\n                hexsha_before = ds.repo.get_hexsha()\n                ds.addurls(self.json_file, \"{url}\",\n                           \"{subdir}-\" + label + \"//{name}\",\n                           save=save)\n                hexsha_after = ds.repo.get_hexsha()\n\n                for fname in [\"foo-{}/a\", \"bar-{}/b\", \"foo-{}/c\"]:\n                    ok_exists(fname.format(label))\n\n                assert_true(save ^ (hexsha_before == hexsha_after))\n                assert_true(save ^ ds.repo.dirty)\n\n            # Now save the \"--nosave\" changes and check that we have\n            # all the subdatasets.\n            ds.add(\".\")\n            eq_(set(subdatasets(ds, recursive=True,\n                                result_xfm=\"relpaths\")),\n                {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file,\n                          \"{url}/nofilename/\",\n                          \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with chpwd(path), patch.object(ds.repo, 'set_metadata', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path), swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[ds.repo.WEB_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\")\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 352,
                                "end_lineno": 576,
                                "parent": null
                            }
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "a0809ed8038e8effbf2c6e6c603ed1263f53a06f^",
    "new_commit_hash": "a0809ed8038e8effbf2c6e6c603ed1263f53a06f",
    "commit_message": "Merge pull request #3561 from kyleam/addurls-path-logic\n\naddurls: Fixes to subpath handling",
    "commit_date": "2019-07-26T12:09:17-04:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "a0809ed8038e8effbf2c6e6c603ed1263f53a06f",
    "test_file_codes": [
        "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport os.path as op\nimport tempfile\n\nfrom mock import patch\n\nfrom six.moves import StringIO\n\nfrom datalad.api import addurls, Dataset, subdatasets\nimport datalad.plugin.addurls as au\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.tests.utils import chpwd, slow, swallow_logs\nfrom datalad.tests.utils import assert_false, assert_true, assert_raises\nfrom datalad.tests.utils import assert_in, assert_re_in, assert_in_results\nfrom datalad.tests.utils import assert_dict_equal\nfrom datalad.tests.utils import eq_, ok_exists\nfrom datalad.tests.utils import create_tree, with_tempfile, HTTPPath\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = au.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = au.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = au.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = au.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = au.Formatter({})\n    eq_(fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}), \"value0\")\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = au.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    eq_(fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\"), \"'x'\")\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    eq_(fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\"), \"x    \")\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = au.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = au.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], {}),\n                         ([\" field=yes \"], {\"field\": \"yes\"}),\n                         ([\"field= value=\"], {\"field\": \"value=\"})]:\n        eq_(au.clean_meta_args(args), expect)\n\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"noequal\"])\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"=value\"])\n\n\ndef test_get_subpaths():\n    for fname, expect in [\n            (op.join(\"no\", \"dbl\", \"slash\"),\n             (op.join(\"no\", \"dbl\", \"slash\"), [])),\n            (\"p1//n\",\n             (op.join(\"p1\", \"n\"), [\"p1\"])),\n            (op.join(\"p1//p2\", \"p3//n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"n\"),\n              [\"p1\", op.join(\"p1\", \"p2\", \"p3\")])),\n            (op.join(\"p1//p2\", \"p3//p4\", \"p5//\", \"n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\", \"n\"),\n              [\"p1\",\n               op.join(\"p1\", \"p2\", \"p3\"),\n               op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\")])),\n            (\"//n\", (op.sep + \"n\", [\"\"])),\n            (\"n//\", (\"n\" + op.sep, [\"n\"]))]:\n        eq_(au.get_subpaths(fname), expect)\n\n\ndef test_sort_paths():\n    paths = [op.join(\"x\", \"a\", \"b\"),\n             \"z\",\n             op.join(\"y\", \"b\"),\n             op.join(\"y\", \"a\")]\n    expected = [\"z\",\n                op.join(\"y\", \"a\"),\n                op.join(\"y\", \"b\"),\n                op.join(\"x\", \"a\", \"b\")]\n    eq_(list(au.sort_paths(paths)), expected)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(au.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(au.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(au.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(au.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(au.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(au.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(au.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(au.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(au.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(au.fmt_to_name(\"\", {}))\n    assert_false(au.fmt_to_name(\"nonames\", {}))\n    assert_false(au.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_file_parts():\n    assert_dict_equal(au.get_file_parts(\"file.tar.gz\", \"prefix\"),\n                      {\"prefix\": \"file.tar.gz\",\n                       \"prefix_root_py\": \"file.tar\",\n                       \"prefix_ext_py\": \".gz\",\n                       \"prefix_root\": \"file\",\n                       \"prefix_ext\": \".tar.gz\"})\n\n\ndef test_get_url_parts():\n    eq_(au.get_url_parts(\"\"), {})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org\"),\n                      {\"_url_hostname\": \"datalad.org\"})\n\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      {\"_url_hostname\": \"datalad.org\",\n                       \"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\",\n                       \"_url_basename_root_py\": \"about\",\n                       \"_url_basename_ext_py\": \".html\",\n                       \"_url_basename_root\": \"about\",\n                       \"_url_basename_ext\": \".html\"})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      au.get_url_parts(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        au.get_url_parts(\"http://datalad.org/for/git-users\"),\n        {\"_url_hostname\": \"datalad.org\",\n         \"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\",\n         \"_url_basename_root_py\": \"git-users\",\n         \"_url_basename_ext_py\": \"\",\n         \"_url_basename_root\": \"git-users\",\n         \"_url_basename_ext\": \"\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\ndef test_extract():\n    info, subpaths = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        [\"adult\", \"kid\", \"adult/no\", \"adult/yes\", \"kid/no\"])\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [\"kid/no/will.csv\", \"adult/yes/bob.csv\",\n         \"adult/no/scott.csv\", \"kid/no/max.csv\"])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [\"kid/no\", \"adult/yes\", \"adult/no\", \"kid/no\"])\n\n\ndef test_extract_disable_autometa():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"*\",\n        meta=[\"group={age_group}\"])\n\n    eq_([d[\"meta_args\"] for d in info],\n        [{\"group\": \"kid\"}, {\"group\": \"adult\"}, {\"group\": \"adult\"},\n         {\"group\": \"kid\"}])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"ea\")\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\"},\n               {\"name\": \"max\", \"age_group\": \"kid\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n\ndef test_extract_csv_json_equal():\n    keys = ST_DATA[\"header\"]\n    csv_rows = [\",\".join(keys)]\n    csv_rows.extend(\",\".join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    kwds = dict(filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n                url_format=\"{name}_{debut_season}.com\",\n                meta=[\"group={age_group}\"])\n\n    json_output = au.extract(json_stream(ST_DATA[\"rows\"]), \"json\", **kwds)\n    csv_output = au.extract(csv_rows, \"csv\", **kwds)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  au.extract, None, \"not_csv_or_json\")\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, no_annex=True)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.addurls(\"dummy_arg0\", \"dummy_arg1\", \"dummy_arg2\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    with chpwd(path):\n        json_file = \"links.json\"\n        with open(json_file, \"w\") as jfh:\n            json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                       {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                       {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                      jfh)\n\n        ds.add(\".\", message=\"setup\")\n\n        with swallow_logs(new_level=logging.INFO) as cml:\n            ds.addurls(json_file,\n                       \"{url}\",\n                       \"{subdir}//{_url_filename_root}\",\n                       dry_run=True)\n\n            for dir_ in [\"foo\", \"bar\"]:\n                assert_in(\"Would create a subdataset at {}\".format(dir_),\n                          cml.out)\n            assert_in(\n                \"Would download URL/a.dat to {}\".format(\n                    os.path.join(path, \"foo\", \"BASE\")),\n                cml.out)\n\n            assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                      cml.out)\n\n\n@slow  # ~9s\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(\n                [{\"url\": cls.url + \"udir/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                 {\"url\": cls.url + \"udir/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                 {\"url\": cls.url + \"udir/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return int(\n                ds.repo.repo.git.rev_list(\"--count\", \"git-annex\").strip())\n\n        n_annex_commits = get_annex_commit_counts()\n\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            filenames = [\"a\", \"b\", \"c\"]\n            for fname in filenames:\n                ok_exists(fname)\n\n            for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                             [\"foo\", \"bar\", \"foo\"]):\n                assert_dict_equal(meta,\n                                  {\"subdir\": [subdir], \"name\": [fname]})\n\n            # Ignore this check if we're faking dates because that disables\n            # batch mode.\n            if not os.environ.get('DATALAD_FAKE__DATES'):\n                # We should have two new commits on the git-annex: one for the\n                # added urls and one for the added metadata.\n                eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n            # Add to already existing links, overwriting.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           ifexists=\"overwrite\")\n                for fname in filenames:\n                    assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                              cml.out)\n\n            # Add to already existing links, skipping.\n            assert_in_results(\n                ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n                action=\"addurls\",\n                status=\"notneeded\")\n\n            # Add to already existing links works, as long content is the same.\n            ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n            # But it fails if something has changed.\n            ds.unlock(\"a\")\n            with open(\"a\", \"w\") as ofh:\n                ofh.write(\"changed\")\n            ds.add(\"a\")\n\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            for save in True, False:\n                label = \"save\" if save else \"nosave\"\n                hexsha_before = ds.repo.get_hexsha()\n                ds.addurls(self.json_file, \"{url}\",\n                           \"{subdir}-\" + label + \"//{name}\",\n                           save=save)\n                hexsha_after = ds.repo.get_hexsha()\n\n                for fname in [\"foo-{}/a\", \"bar-{}/b\", \"foo-{}/c\"]:\n                    ok_exists(fname.format(label))\n\n                assert_true(save ^ (hexsha_before == hexsha_after))\n                assert_true(save ^ ds.repo.dirty)\n\n            # Now save the \"--nosave\" changes and check that we have\n            # all the subdatasets.\n            ds.add(\".\")\n            eq_(set(subdatasets(ds, recursive=True,\n                                result_xfm=\"relpaths\")),\n                {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n            # We don't try to recreate existing subdatasets.\n            with swallow_logs(new_level=logging.DEBUG) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n                assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with chpwd(path):\n            with assert_raises(IncompleteResultsError) as raised:\n                ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n            assert_in(\"There are file name collisions\", str(raised.exception))\n\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n            for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n            for fname in [\"udir/a.dat\", \"udir/b.dat\", \"udir/c.dat\"]:\n                ok_exists(fname)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path):\n            assert_raises(IncompleteResultsError,\n                          ds.addurls,\n                          self.json_file,\n                          \"{url}/nofilename/\",\n                          \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with chpwd(path), patch.object(ds.repo, 'set_metadata', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with chpwd(path), swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[ds.repo.WEB_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\")\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.11.5.dev60\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 42ms\nInstalled 25 packages in 5.86s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 66 packages in 4.43s\nPrepared 1 package in 4.33s\nInstalled 60 packages in 279ms\n + appdirs==1.4.4\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.11.5 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_a0809ed8038e8effbf2c6e6c603ed1263f53a06f)\n + deprecated==1.2.15\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + gitdb==4.0.12\n + gitpython==3.1.44\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.15.0\n + pillow==11.0.0\n + propcache==0.2.1\n + pycparser==2.22\n + pygithub==2.5.0\n + pyjwt==2.10.1\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.2\n + soupsieve==2.6\n + tqdm==4.67.1\n + urllib3==1.26.20\n + vcrpy==7.0.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 99 packages in 338ms\nInstalled 31 packages in 718ms\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + filelock==3.16.1\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + nose-timer==1.0.1\n + platformdirs==4.3.6\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_a0809ed8038e8effbf2c6e6c603ed1263f53a06f\nplugins: cov-6.0.0\ncollected 34 items\n\nr2e_tests/test_1.py .....................FFFFFFFFFFFFF                   [100%]\n\n=================================== FAILURES ===================================\n__________________________ test_addurls_nonannex_repo __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexeqja2x2k'}\nfilename = '/tmp/datalad_temp_test_addurls_nonannex_repo7umvs_po'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_nonannex_repo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_____________________________ test_addurls_dry_run _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex6a6hwdko'}\nfilename = '/tmp/datalad_temp_test_addurls_dry_run8mm4_b85'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dry_run() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n___________________________ TestAddurls.test_addurls ___________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262046e5b0>,)\nkw = {'path': '/tmp/tmp-page2annexzws2ifwi'}\nfilename = '/tmp/datalad_temp_test_addurlsn1b4yfb4'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n__________________ TestAddurls.test_addurls_create_newdataset __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262046e970>,)\nkw = {'path': '/tmp/tmp-page2annex1oc503j1'}\nfilename = '/tmp/datalad_temp_test_addurls_create_newdataset0bxv2y2l'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_create_newdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_____________________ TestAddurls.test_addurls_subdataset ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262049fac0>,)\nkw = {'path': '/tmp/tmp-page2annexj1_si_0u'}\nfilename = '/tmp/datalad_temp_test_addurls_subdatasetwa7i81nw'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_subdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n______________________ TestAddurls.test_addurls_repindex _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f2620640580>,)\nkw = {'path': '/tmp/tmp-page2annexvac9c_qz'}\nfilename = '/tmp/datalad_temp_test_addurls_repindexi2y83kms'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_repindex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n______________________ TestAddurls.test_addurls_url_parts ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f26206407c0>,)\nkw = {'path': '/tmp/tmp-page2annexgbmde_f4'}\nfilename = '/tmp/datalad_temp_test_addurls_url_partstbhuf8li'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_parts() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n____________________ TestAddurls.test_addurls_url_filename _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262034b040>,)\nkw = {'path': '/tmp/tmp-page2annex54wps2yf'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filename86a3xhsi'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n__________________ TestAddurls.test_addurls_url_filename_fail __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262046e8e0>,)\nkw = {'path': '/tmp/tmp-page2annexlbbawnpp'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filename_failoghizj_d'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename_fail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n______________________ TestAddurls.test_addurls_metafail _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262046efd0>,)\nkw = {'path': '/tmp/tmp-page2annexj0dnor2x'}\nfilename = '/tmp/datalad_temp_test_addurls_metafail6fxjatku'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_metafail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n____________________ TestAddurls.test_addurls_dropped_urls _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262046ee80>,)\nkw = {'path': '/tmp/tmp-page2annexdsxra37d'}\nfilename = '/tmp/datalad_temp_test_addurls_dropped_urls7jrf09md'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dropped_urls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_______________________ TestAddurls.test_addurls_version _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f262046eca0>,)\nkw = {'path': '/tmp/tmp-page2annex7qkgotal'}\nfilename = '/tmp/datalad_temp_test_addurls_version1v50dfwh'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_version() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_______________________ TestAddurls.test_addurls_deeper ________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7f26205369d0>,)\nkw = {'path': '/tmp/tmp-page2annex8iddpvku'}\nfilename = '/tmp/datalad_temp_test_addurls_deeperzt97ivhj'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_deeper() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_lower_case\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_formatter_missing_arg\nPASSED r2e_tests/test_1.py::test_repformatter\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_get_subpaths\nPASSED r2e_tests/test_1.py::test_sort_paths\nPASSED r2e_tests/test_1.py::test_is_legal_metafield\nPASSED r2e_tests/test_1.py::test_filter_legal_metafield\nPASSED r2e_tests/test_1.py::test_fmt_to_name\nPASSED r2e_tests/test_1.py::test_get_file_parts\nPASSED r2e_tests/test_1.py::test_get_url_parts\nPASSED r2e_tests/test_1.py::test_extract\nPASSED r2e_tests/test_1.py::test_extract_disable_autometa\nPASSED r2e_tests/test_1.py::test_extract_exclude_autometa_regexp\nPASSED r2e_tests/test_1.py::test_extract_csv_json_equal\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\nFAILED r2e_tests/test_1.py::test_addurls_nonannex_repo - TypeError: test_addu...\nFAILED r2e_tests/test_1.py::test_addurls_dry_run - TypeError: test_addurls_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls - TypeError: test_addur...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_create_newdataset - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_subdataset - TypeError:...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_repindex - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_parts - TypeError: ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename_fail - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_metafail - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_dropped_urls - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_version - TypeError: te...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_deeper - TypeError: tes...\n======================== 13 failed, 21 passed in 2.80s =========================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_a0809ed8038e8effbf2c6e6c603ed1263f53a06f\nplugins: cov-6.0.0\ncollected 34 items\n\nr2e_tests/test_1.py .........FF.....F....FFFFFFFFFFFFF                   [100%]\n\n=================================== FAILURES ===================================\n______________________________ test_get_subpaths _______________________________\n\n    def test_get_subpaths():\n        for fname, expect in [\n                (op.join(\"no\", \"dbl\", \"slash\"),\n                 (op.join(\"no\", \"dbl\", \"slash\"), [])),\n                (\"p1//n\",\n                 (op.join(\"p1\", \"n\"), [\"p1\"])),\n                (op.join(\"p1//p2\", \"p3//n\"),\n                 (op.join(\"p1\", \"p2\", \"p3\", \"n\"),\n                  [\"p1\", op.join(\"p1\", \"p2\", \"p3\")])),\n                (op.join(\"p1//p2\", \"p3//p4\", \"p5//\", \"n\"),\n                 (op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\", \"n\"),\n                  [\"p1\",\n                   op.join(\"p1\", \"p2\", \"p3\"),\n                   op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\")])),\n                (\"//n\", (op.sep + \"n\", [\"\"])),\n                (\"n//\", (\"n\" + op.sep, [\"n\"]))]:\n>           eq_(au.get_subpaths(fname), expect)\n\nr2e_tests/test_1.py:136: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = ('p1/p2/p3/p4/p5/n', ['p1', 'p1/p2/p3', 'p1/p1/p2/p3/p4/p5'])\nb = ('p1/p2/p3/p4/p5/n', ['p1', 'p1/p2/p3', 'p1/p2/p3/p4/p5']), msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: ('p1/p2/p3/p4/p5/n', ['p1', 'p1/p2/p3', 'p1/p1/p2/p3/p4/p5']) != ('p1/p2/p3/p4/p5/n', ['p1', 'p1/p2/p3', 'p1/p2/p3/p4/p5'])\n\n.venv/lib/python3.9/site-packages/nose/tools/trivial.py:29: AssertionError\n_______________________________ test_sort_paths ________________________________\n\n    def test_sort_paths():\n        paths = [op.join(\"x\", \"a\", \"b\"),\n                 \"z\",\n                 op.join(\"y\", \"b\"),\n                 op.join(\"y\", \"a\")]\n        expected = [\"z\",\n                    op.join(\"y\", \"a\"),\n                    op.join(\"y\", \"b\"),\n                    op.join(\"x\", \"a\", \"b\")]\n>       eq_(list(au.sort_paths(paths)), expected)\nE       AttributeError: module 'datalad.plugin.addurls' has no attribute 'sort_paths'\n\nr2e_tests/test_1.py:148: AttributeError\n_________________________________ test_extract _________________________________\n\n    def test_extract():\n        info, subpaths = au.extract(\n            json_stream(ST_DATA[\"rows\"]), \"json\",\n            url_format=\"{name}_{debut_season}.com\",\n            filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n    \n>       eq_(subpaths,\n            [\"adult\", \"kid\", \"adult/no\", \"adult/yes\", \"kid/no\"])\n\nr2e_tests/test_1.py:237: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = {'adult', 'adult/no', 'adult/yes', 'kid', 'kid/no'}\nb = ['adult', 'kid', 'adult/no', 'adult/yes', 'kid/no'], msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: {'adult/yes', 'kid', 'adult/no', 'adult', 'kid/no'} != ['adult', 'kid', 'adult/no', 'adult/yes', 'kid/no']\n\n.venv/lib/python3.9/site-packages/nose/tools/trivial.py:29: AssertionError\n__________________________ test_addurls_nonannex_repo __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexatdbf8_v'}\nfilename = '/tmp/datalad_temp_test_addurls_nonannex_repo9t0t5ggj'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_nonannex_repo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_____________________________ test_addurls_dry_run _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexbnso1vxh'}\nfilename = '/tmp/datalad_temp_test_addurls_dry_run_u0qs17x'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dry_run() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n___________________________ TestAddurls.test_addurls ___________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee56a970>,)\nkw = {'path': '/tmp/tmp-page2annexdxlge0c3'}\nfilename = '/tmp/datalad_temp_test_addurlsgzyf__4l'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n__________________ TestAddurls.test_addurls_create_newdataset __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee56ab50>,)\nkw = {'path': '/tmp/tmp-page2annexz91a3jnu'}\nfilename = '/tmp/datalad_temp_test_addurls_create_newdataset40yxoadk'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_create_newdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_____________________ TestAddurls.test_addurls_subdataset ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee56ad30>,)\nkw = {'path': '/tmp/tmp-page2annexci4bvoe8'}\nfilename = '/tmp/datalad_temp_test_addurls_subdatasetf6nf3fox'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_subdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n______________________ TestAddurls.test_addurls_repindex _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee56af10>,)\nkw = {'path': '/tmp/tmp-page2annexgg942o5n'}\nfilename = '/tmp/datalad_temp_test_addurls_repindex7c09ffrx'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_repindex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n______________________ TestAddurls.test_addurls_url_parts ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee576130>,)\nkw = {'path': '/tmp/tmp-page2annex5he5ixnz'}\nfilename = '/tmp/datalad_temp_test_addurls_url_partsr5rn6h6k'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_parts() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n____________________ TestAddurls.test_addurls_url_filename _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee576310>,)\nkw = {'path': '/tmp/tmp-page2annexzzv0toiu'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filenamejp8zy7ns'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n__________________ TestAddurls.test_addurls_url_filename_fail __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee5764f0>,)\nkw = {'path': '/tmp/tmp-page2annex9skgebft'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filename_fail63690t8r'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename_fail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n______________________ TestAddurls.test_addurls_metafail _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee5766d0>,)\nkw = {'path': '/tmp/tmp-page2annexpps6fdu5'}\nfilename = '/tmp/datalad_temp_test_addurls_metafailvp3zwo2g'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_metafail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n____________________ TestAddurls.test_addurls_dropped_urls _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee5768b0>,)\nkw = {'path': '/tmp/tmp-page2annexp9kn4b4p'}\nfilename = '/tmp/datalad_temp_test_addurls_dropped_urlsfplof_3_'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dropped_urls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_______________________ TestAddurls.test_addurls_version _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee576ac0>,)\nkw = {'path': '/tmp/tmp-page2annex9h0uaobu'}\nfilename = '/tmp/datalad_temp_test_addurls_versionjw88r1jd'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_version() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n_______________________ TestAddurls.test_addurls_deeper ________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x73f7ee576cd0>,)\nkw = {'path': '/tmp/tmp-page2annex08dy9482'}\nfilename = '/tmp/datalad_temp_test_addurls_deeperkucz0vhz'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_deeper() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:615: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_lower_case\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_formatter_missing_arg\nPASSED r2e_tests/test_1.py::test_repformatter\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_is_legal_metafield\nPASSED r2e_tests/test_1.py::test_filter_legal_metafield\nPASSED r2e_tests/test_1.py::test_fmt_to_name\nPASSED r2e_tests/test_1.py::test_get_file_parts\nPASSED r2e_tests/test_1.py::test_get_url_parts\nPASSED r2e_tests/test_1.py::test_extract_disable_autometa\nPASSED r2e_tests/test_1.py::test_extract_exclude_autometa_regexp\nPASSED r2e_tests/test_1.py::test_extract_csv_json_equal\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\nFAILED r2e_tests/test_1.py::test_get_subpaths - AssertionError: ('p1/p2/p3/p4...\nFAILED r2e_tests/test_1.py::test_sort_paths - AttributeError: module 'datalad...\nFAILED r2e_tests/test_1.py::test_extract - AssertionError: {'adult/yes', 'kid...\nFAILED r2e_tests/test_1.py::test_addurls_nonannex_repo - TypeError: test_addu...\nFAILED r2e_tests/test_1.py::test_addurls_dry_run - TypeError: test_addurls_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls - TypeError: test_addur...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_create_newdataset - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_subdataset - TypeError:...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_repindex - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_parts - TypeError: ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename_fail - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_metafail - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_dropped_urls - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_version - TypeError: te...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_deeper - TypeError: tes...\n======================== 16 failed, 18 passed in 1.61s =========================\n",
    "old_commit_res_stderr": ""
}
datalad/plugin/addurls.py datalad/plugin/tests/test_addurls.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
### Incorrect Subpath Construction Causes Path Duplication and Sorting Errors

**Description**
When processing filenames with multiple subpath delimiters, the system incorrectly constructs the subpaths, resulting in duplicated or malformed paths. This leads to inconsistencies and failures in downstream operations that rely on correctly structured subpaths.

**Example Buggy Code**
```python
filename = "p1//p2/p3//n"
subpaths = get_subpaths(filename)
print(subpaths)  # Output: ['p1', 'p1/p1/p2/p3']
```
**Expected Behavior**
- Subpaths should be constructed without duplication, accurately reflecting the intended directory structure.
- Subpaths should be sorted by directory depth and alphabetically to ensure consistent processing.

**Actual Behavior**
- Subpaths are constructed with duplicated path segments, e.g., `['p1', 'p1/p1/p2/p3']` instead of `['p1', 'p1/p2/p3']`.
- Attempting to sort subpaths results in errors due to missing sorting functionality.

This issue affects the reliability of subpath handling within the addurls plugin, causing related tests to fail and potentially leading to incorrect data organization.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import re
import string

from six import string_types
from six.moves.urllib.parse import urlparse

from datalad.dochelpers import exc_str
from datalad.log import log_progress, with_result_progress
from datalad.interface.base import Interface
from datalad.interface.base import build_doc
from datalad.interface.results import annexjson2result, get_status_dict
from datalad.interface.common_opts import nosave_opt
from datalad.support.exceptions import AnnexBatchCommandError
from datalad.support.network import get_url_filename
from datalad.support.path import split_ext
from datalad.support.s3 import get_versioned_url
from datalad.utils import (
    assure_list,
    unlink,
)

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    missing : str, optional
        When column lookup results in an empty string, use this value in
        its place.
    """

    def __init__(self, idx_to_name=None, missing_value=None):
        self.idx_to_name = idx_to_name or {}
        self.missing = missing_value

    def format(self, format_string, *args, **kwargs):
        if not isinstance(args[0], Mapping):
            raise ValueError("First positional argument should be mapping")
        return super(Formatter, self).format(format_string, *args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]

        name = key
        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            name = self.idx_to_name[key_int]

        try:
            value = data[name]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)

        if self.missing is not None and isinstance(value, string_types):
            return value or self.missing
        return value

    def convert_field(self, value, conversion):
        if conversion == 'l':
            return str(value).lower()
        return super(Formatter, self).convert_field(value, conversion)


class RepFormatter(Formatter):
    """Extend Formatter to support a {_repindex} placeholder.
    """

    def __init__(self, *args, **kwargs):
        super(RepFormatter, self).__init__(*args, **kwargs)
        self.repeats = {}
        self.repindex = 0

    def format(self, *args, **kwargs):
        self.repindex = 0
        result = super(RepFormatter, self).format(*args, **kwargs)
        if result in self.repeats:
            self.repindex = self.repeats[result] + 1
            self.repeats[result] = self.repindex
            result = super(RepFormatter, self).format(*args, **kwargs)
        else:
            self.repeats[result] = 0
        return result

    def get_value(self, key, args, kwargs):
        args[0]["_repindex"] = self.repindex
        return super(RepFormatter, self).get_value(key, args, kwargs)


def clean_meta_args(args):
    """Process metadata arguments.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    A dict mapping field names to values.
    """
    results = {}
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            raise ValueError("meta argument isn't in 'field=value' format")

        if not value:
            # The `url_file` may have an empty value.
            continue
        results[field] = value
    return results


def get_subpaths(filename):
    """Convert "//" marker in `filename` to a list of subpaths.

    >>> from datalad.plugin.addurls import get_subpaths
    >>> get_subpaths("p1/p2//p3/p4//file")
    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])

    Note: With Python 3, the subpaths could be generated with

        itertools.accumulate(filename.split("//")[:-1], os.path.join)

    Parameters
    ----------
    filename : str
        File name with "//" marking subpaths.

    Returns
    -------
    A tuple of the filename with any "//" collapsed to a single
    separator and a list of subpaths (str).
    """
    if "//" not in filename:
        return filename, []

    spaths = []
    for part in filename.split("//")[:-1]:
        path = os.path.join(*(spaths + [part]))
        spaths.append(path)
    return filename.replace("//", os.path.sep), spaths


def is_legal_metafield(name):
    """Test whether `name` is a valid metadata field.

    The set of permitted characters is taken from git-annex's
    MetaData.hs:legalField.
    """
    return bool(re.match(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z", name))


def filter_legal_metafield(fields):
    """Remove illegal names from `fields`.

    Note: This is like `filter(is_legal_metafield, fields)` but the
    dropped values are logged.
    """
    legal = []
    for field in fields:
        if is_legal_metafield(field):
            legal.append(field)
        else:
            lgr.debug("%s is not a valid metadata field name; dropping",
                      field)
    return legal


def get_fmt_names(format_string):
    """Yield field names in `format_string`.
    """
    for _, name, _, _ in string.Formatter().parse(format_string):
        if name:
            yield name


def fmt_to_name(format_string, num_to_name):
    """Try to map a format string to a single name.

    Parameters
    ----------
    format_string : string
    num_to_name : dict
        A dictionary that maps from an integer to a column name.  This
        enables mapping the format string to an integer to a name.

    Returns
    -------
    A placeholder name if `format_string` consists of a single
    placeholder and no other text.  Otherwise, None is returned.
    """
    parsed = list(string.Formatter().parse(format_string))
    if len(parsed) != 1:
        # It's an empty string or there's more than one placeholder.
        return
    if parsed[0][0]:
        # Format string contains text before the placeholder.
        return

    name = parsed[0][1]
    if not name:
        # The field name is empty.
        return

    try:
        return num_to_name[int(name)]
    except (KeyError, ValueError):
        return name


def _read(stream, input_type):
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        idx_map = dict(enumerate(headers))
        rows = [dict(zip(headers, r)) for r in csvrows]
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        idx_map = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")
    return rows, idx_map


def _format_filenames(format_fn, rows, row_infos):
    subpaths = set()
    for row, info in zip(rows, row_infos):
        filename = format_fn(row)
        filename, spaths = get_subpaths(filename)
        subpaths |= set(spaths)
        info["filename"] = filename
        info["subpath"] = spaths[-1] if spaths else None
    return subpaths


def get_file_parts(filename, prefix="name"):
    """Assign a name to various parts of a file.

    Parameters
    ----------
    filename : str
        A file name (no leading path is permitted).
    prefix : str
        Prefix to prepend to the key names.

    Returns
    -------
    A dict mapping each part to a value.
    """
    root, ext = split_ext(filename)
    root_py, ext_py = os.path.splitext(filename)

    return {prefix: filename,
            prefix + "_root": root,
            prefix + "_ext": ext,
            prefix + "_root_py": root_py,
            prefix + "_ext_py": ext_py}


def get_url_parts(url):
    """Assign a name to various parts of the URL.

    Parameters
    ----------
    url : str

    Returns
    -------
    A dict with keys `_url_hostname` and, for a path with N+1 parts,
    '_url0' through '_urlN' .  There is also a `_url_basename` key for
    the rightmost part of the path.
    """
    parsed = urlparse(url)
    if not parsed.netloc:
        return {}

    names = {"_url_hostname": parsed.netloc}

    path = parsed.path.strip("/")
    if not path:
        return names

    url_parts = path.split("/")
    for pidx, part in enumerate(url_parts):
        names["_url{}".format(pidx)] = part
    basename = url_parts[-1]
    names["_url_basename"] = basename
    names.update(get_file_parts(basename, prefix="_url_basename"))
    return names


def add_extra_filename_values(filename_format, rows, urls, dry_run):
    """Extend `rows` with values for special formatting fields.
    """
    file_fields = list(get_fmt_names(filename_format))
    if any(i.startswith("_url") for i in file_fields):
        for row, url in zip(rows, urls):
            row.update(get_url_parts(url))

    if any(i.startswith("_url_filename") for i in file_fields):
        if dry_run:  # Don't waste time making requests.
            dummy = get_file_parts("BASE.EXT", "_url_filename")
            for idx, row in enumerate(rows):
                row.update(
                    {k: v + str(idx) for k, v in dummy.items()})
        else:
            num_urls = len(urls)
            log_progress(lgr.info, "addurls_requestnames",
                         "Requesting file names for %d URLs", num_urls,
                         label="Requesting names", total=num_urls,
                         unit=" Files")
            for row, url in zip(rows, urls):
                # If we run into any issues here, we're just going to raise an
                # exception and then abort inside dlplugin.  It'd be good to
                # disentangle this from `extract` so that we could yield an
                # individual error, drop the row, and keep going.
                filename = get_url_filename(url)
                if filename:
                    row.update(get_file_parts(filename, "_url_filename"))
                else:
                    raise ValueError(
                        "{} does not contain a filename".format(url))
                log_progress(lgr.info, "addurls_requestnames",
                             "%s returned for %s", url, filename,
                             update=1, increment=True)
            log_progress(lgr.info, "addurls_requestnames",
                         "Finished requesting file names")


def extract(stream, input_type, url_format="{0}", filename_format="{1}",
            exclude_autometa=None, meta=None,
            dry_run=False, missing_value=None):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.
    input_type : {'csv', 'json'}

    All other parameters match those described in `AddUrls`.

    Returns
    -------
    A tuple where the first item is a list with a dict of extracted information
    for each row in `stream` and the second item is a set that contains all the
    subdataset paths.
    """
    meta = assure_list(meta)

    rows, colidx_to_name = _read(stream, input_type)

    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta
    format_url = partial(fmt.format, url_format)

    auto_meta_args = []
    if exclude_autometa not in ["*", ""]:
        urlcol = fmt_to_name(url_format, colidx_to_name)
        # TODO: Try to normalize invalid fields, checking for any
        # collisions.
        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)
        if exclude_autometa:
            metacols = (c for c in metacols
                        if not re.search(exclude_autometa, c))
        metacols = filter_legal_metafield(metacols)
        auto_meta_args = [c + "=" + "{" + c + "}" for c in metacols]

    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]

    rows_with_url = []
    infos = []
    for row in rows:
        url = format_url(row)
        if not url or url == missing_value:
            continue  # pragma: no cover, peephole optimization
        rows_with_url.append(row)
        meta_args = clean_meta_args(fmt(row) for fmt in formats_meta)
        infos.append({"url": url, "meta_args": meta_args})

    n_dropped = len(rows) - len(rows_with_url)
    if n_dropped:
        lgr.warning("Dropped %d row(s) that had an empty URL", n_dropped)

    # Format the filename in a second pass so that we can provide
    # information about the formatted URLs.
    add_extra_filename_values(filename_format, rows_with_url,
                              [i["url"] for i in infos],
                              dry_run)

    # For the file name, we allow the _repindex special key.
    format_filename = partial(
        RepFormatter(colidx_to_name, missing_value).format,
        filename_format)
    subpaths = _format_filenames(format_filename, rows_with_url, infos)
    return infos, subpaths


@with_result_progress("Adding URLs")
def add_urls(rows, ifexists=None, options=None):
    """Call `git annex addurl` using information in `rows`.
    """
    for row in rows:
        filename_abs = row["filename_abs"]
        ds, filename = row["ds"], row["ds_filename"]
        lgr.debug("Adding metadata to %s in %s", filename, ds.path)

        if os.path.exists(filename_abs) or os.path.islink(filename_abs):
            if ifexists == "skip":
                yield get_status_dict(action="addurls",
                                      ds=ds,
                                      type="file",
                                      path=filename_abs,
                                      status="notneeded")
                continue
            elif ifexists == "overwrite":
                lgr.debug("Removing %s", filename_abs)
                unlink(filename_abs)
            else:
                lgr.debug("File %s already exists", filename_abs)

        try:
            out_json = ds.repo.add_url_to_file(filename, row["url"],
                                               batch=True, options=options)
        except AnnexBatchCommandError as exc:
            yield get_status_dict(action="addurls",
                                  ds=ds,
                                  type="file",
                                  path=filename_abs,
                                  message=exc_str(exc),
                                  status="error")
            continue

        # In the case of an error, the json object has file=None.
        if out_json["file"] is None:
            out_json["file"] = filename_abs
        yield annexjson2result(out_json, ds, action="addurls",
                               type="file", logger=lgr)


@with_result_progress("Adding metadata")
def add_meta(rows):
    """Call `git annex metadata --set` using information in `rows`.
    """
    from mock import patch

    for row in rows:
        ds, filename = row["ds"], row["ds_filename"]

        with patch.object(ds.repo, "always_commit", False):
            lgr.debug("Adding metadata to %s in %s", filename, ds.path)
            for a in ds.repo.set_metadata(filename, add=row["meta_args"]):
                res = annexjson2result(a, ds, type="file", logger=lgr)
                # Don't show all added metadata for the file because that
                # could quickly flood the output.
                del res["message"]
                yield res


@build_doc
class Addurls(Interface):
    """Create and update a dataset from a list of URLs.

    *Format specification*

    Several arguments take format strings.  These are similar to normal Python
    format strings where the names from `URL-FILE` (column names for a CSV or
    properties for JSON) are available as placeholders.  If `URL-FILE` is a CSV
    file, a positional index can also be used (i.e., "{0}" for the first
    column).  Note that a placeholder cannot contain a ':' or '!'.

    In addition, the `FILENAME-FORMAT` arguments has a few special
    placeholders.

      - _repindex

        The constructed file names must be unique across all fields rows.  To
        avoid collisions, the special placeholder "_repindex" can be added to
        the formatter.  Its value will start at 0 and increment every time a
        file name repeats.

      - _url_hostname, _urlN, _url_basename*

        Various parts of the formatted URL are available.  Take
        "http://datalad.org/asciicast/seamless_nested_repos.sh" as an example.

        "datalad.org" is stored as "_url_hostname".  Components of the URL's
        path can be referenced as "_urlN".  "_url0" and "_url1" would map to
        "asciicast" and "seamless_nested_repos.sh", respectively.  The final
        part of the path is also available as "_url_basename".

        This name is broken down further.  "_url_basename_root" and
        "_url_basename_ext" provide access to the root name and extension.
        These values are similar to the result of os.path.splitext, but, in the
        case of multiple periods, the extension is identified using the same
        length heuristic that git-annex uses.  As a result, the extension of
        "file.tar.gz" would be ".tar.gz", not ".gz".  In addition, the fields
        "_url_basename_root_py" and "_url_basename_ext_py" provide access to
        the result of os.path.splitext.

      - _url_filename*

        These are similar to _url_basename* fields, but they are obtained with
        a server request.  This is useful if the file name is set in the
        Content-Disposition header.


    *Examples*

    Consider a file "avatars.csv" that contains::

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and 'ext'
    fields, we could run::

      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'

    The `-d avatar_ds` is used to create a new dataset in "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new subdataset in an
    "avatars" subdirectory, we could use "//" in the `FILENAME-FORMAT`
    argument::

      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'

    .. note::

       For users familiar with 'git annex addurl': A large part of this
       plugin's functionality can be viewed as transforming data from
       `URL-FILE` into a "url filename" format that fed to 'git annex addurl
       --batch --with-files'.
    """

    from datalad.distribution.dataset import datasetmethod
    from datalad.interface.utils import eval_results
    from datalad.distribution.dataset import EnsureDataset
    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr
    from datalad.support.param import Parameter

    _params_ = dict(
        dataset=Parameter(
            args=("-d", "--dataset"),
            doc="""Add the URLs to this dataset (or possibly subdatasets of
            this dataset).  An empty or non-existent directory is passed to
            create a new dataset.  New subdatasets can be specified with
            `FILENAME-FORMAT`.""",
            constraints=EnsureDataset() | EnsureNone()),
        urlfile=Parameter(
            args=("urlfile",),
            metavar="URL-FILE",
            doc="""A file that contains URLs or information that can be used to
            construct URLs.  Depending on the value of --input-type, this
            should be a CSV file (with a header as the first row) or a JSON
            file (structured as a list of objects with string values)."""),
        urlformat=Parameter(
            args=("urlformat",),
            metavar="URL-FORMAT",
            doc="""A format string that specifies the URL for each entry.  See
            the 'Format Specification' section above."""),
        filenameformat=Parameter(
            args=("filenameformat",),
            metavar="FILENAME-FORMAT",
            doc="""Like `URL-FORMAT`, but this format string specifies the file
            to which the URL's content will be downloaded.  The file name may
            contain directories.  The separator "//" can be used to indicate
            that the left-side directory should be created as a new subdataset.
            See the 'Format Specification' section above."""),
        input_type=Parameter(
            args=("-t", "--input-type"),
            metavar="TYPE",
            doc="""Whether `URL-FILE` should be considered a CSV file or a JSON
            file.  The default value, "ext", means to consider `URL-FILE` as a
            JSON file if it ends with ".json".  Otherwise, treat it as a CSV
            file.""",
            constraints=EnsureChoice("ext", "csv", "json")),
        exclude_autometa=Parameter(
            args=("-x", "--exclude_autometa"),
            metavar="REGEXP",
            doc="""By default, metadata field=value pairs are constructed with
            each column in `URL-FILE`, excluding any single column that is
            specified via `URL-FORMAT`.  This argument can be used to exclude
            columns that match a regular expression.  If set to '*' or an empty
            string, automatic metadata extraction is disabled completely.  This
            argument does not affect metadata set explicitly with --meta."""),
        meta=Parameter(
            args=("-m", "--meta",),
            metavar="FORMAT",
            action="append",
            doc="""A format string that specifies metadata.  It should be
            structured as "<field>=<value>".  As an example, "location={3}"
            would mean that the value for the "location" metadata field should
            be set the value of the fourth column.  This option can be given
            multiple times."""),
        message=Parameter(
            args=("--message",),
            metavar="MESSAGE",
            doc="""Use this message when committing the URL additions.""",
            constraints=EnsureNone() | EnsureStr()),
        dry_run=Parameter(
            args=("-n", "--dry-run"),
            action="store_true",
            doc="""Report which URLs would be downloaded to which files and
            then exit."""),
        fast=Parameter(
            args=("--fast",),
            action="store_true",
            doc="""If True, add the URLs, but don't download their content.
            Underneath, this passes the --fast flag to `git annex addurl`."""),
        ifexists=Parameter(
            args=("--ifexists",),
            metavar="ACTION",
            doc="""What to do if a constructed file name already exists.  The
            default behavior is to proceed with the `git annex addurl`, which
            will fail if the file size has changed.  If set to 'overwrite',
            remove the old file before adding the new one.  If set to 'skip',
            do not add the new file.""",
            constraints=EnsureNone() | EnsureChoice("overwrite", "skip")),
        missing_value=Parameter(
            args=("--missing-value",),
            metavar="VALUE",
            doc="""When an empty string is encountered, use this value
            instead.""",
            constraints=EnsureNone() | EnsureStr()),
        save=nosave_opt,
        version_urls=Parameter(
            args=("--version-urls",),
            action="store_true",
            doc="""Try to add a version ID to the URL. This currently only has
            an effect on URLs for AWS S3 buckets."""),
    )

    @staticmethod
    @datasetmethod(name='addurls')
    @eval_results
    def __call__(dataset, urlfile, urlformat, filenameformat,
                 input_type="ext", exclude_autometa=None, meta=None,
                 message=None, dry_run=False, fast=False, ifexists=None,
                 missing_value=None, save=True, version_urls=False):
        # Temporarily work around gh-2269.
        url_file = urlfile
        url_format, filename_format = urlformat, filenameformat

        from requests.exceptions import RequestException

        from datalad.distribution.dataset import Dataset, require_dataset
        from datalad.interface.results import get_status_dict
        from datalad.support.annexrepo import AnnexRepo

        lgr = logging.getLogger("datalad.plugin.addurls")

        dataset = require_dataset(dataset, check_installed=False)
        if dataset.repo and not isinstance(dataset.repo, AnnexRepo):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message="not an annex repo")
            return

        if input_type == "ext":
            extension = os.path.splitext(url_file)[1]
            input_type = "json" if extension == ".json" else "csv"

        with open(url_file) as fd:
            try:
                rows, subpaths = extract(fd, input_type,
                                         url_format, filename_format,
                                         exclude_autometa, meta,
                                         dry_run,
                                         missing_value)
            except (ValueError, RequestException) as exc:
                yield get_status_dict(action="addurls",
                                      ds=dataset,
                                      status="error",
                                      message=exc_str(exc))
                return

        if len(rows) != len(set(row["filename"] for row in rows)):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message=("There are file name collisions; "
                                           "consider using {_repindex}"))
            return

        if dry_run:
            for subpath in subpaths:
                lgr.info("Would create a subdataset at %s", subpath)
            for row in rows:
                lgr.info("Would download %s to %s",
                         row["url"],
                         os.path.join(dataset.path, row["filename"]))
                lgr.info("Metadata: %s",
                         sorted(u"{}={}".format(k, v)
                                for k, v in row["meta_args"].items()))
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="ok",
                                  message="dry-run finished")
            return

        if not dataset.repo:
            # Populate a new dataset with the URLs.
            for r in dataset.create(result_xfm=None, return_type='generator',
                                    save=save):
                yield r

        annex_options = ["--fast"] if fast else []

        for spath in subpaths:
            if os.path.exists(os.path.join(dataset.path, spath)):
                lgr.warning(
                    "Not creating subdataset at existing path: %s",
                    spath)
            else:
                for r in dataset.create(spath, result_xfm=None,
                                        return_type='generator', save=save):
                    yield r

        for row in rows:
            # Add additional information that we'll need for various
            # operations.
            filename_abs = os.path.join(dataset.path, row["filename"])
            if row["subpath"]:
                ds_current = Dataset(os.path.join(dataset.path,
                                                  row["subpath"]))
                ds_filename = os.path.relpath(filename_abs, ds_current.path)
            else:
                ds_current = dataset
                ds_filename = row["filename"]
            row.update({"filename_abs": filename_abs,
                        "ds": ds_current,
                        "ds_filename": ds_filename})

        if version_urls:
            num_urls = len(rows)
            log_progress(lgr.info, "addurls_versionurls",
                         "Versioning %d URLs", num_urls,
                         label="Versioning URLs",
                         total=num_urls, unit=" URLs")
            for row in rows:
                url = row["url"]
                try:
                    row["url"] = get_versioned_url(url)
                except (ValueError, NotImplementedError) as exc:
                    # We don't expect this to happen because get_versioned_url
                    # should return the original URL if it isn't an S3 bucket.
                    # It only raises exceptions if it doesn't know how to
                    # handle the scheme for what looks like an S3 bucket.
                    lgr.warning("error getting version of %s: %s",
                                row["url"], exc_str(exc))
                log_progress(lgr.info, "addurls_versionurls",
                             "Versioned result for %s: %s", url, row["url"],
                             update=1, increment=True)
            log_progress(lgr.info, "addurls_versionurls", "Finished versioning URLs")

        files_to_add = set()
        for r in add_urls(rows, ifexists=ifexists, options=annex_options):
            if r["status"] == "ok":
                files_to_add.add(r["path"])
            yield r

            msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)

        if files_to_add:
            for r in dataset.add(files_to_add, save=False):
                yield r

            meta_rows = [r for r in rows if r["filename_abs"] in files_to_add]
            for r in add_meta(meta_rows):
                yield r

            # Save here rather than the add call above to trigger a metadata
            # commit on the git-annex branch.
            if save:
                for r in dataset.save(message=msg, recursive=True):
                    yield r


__datalad_plugin__ = Addurls

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
### Incorrect Subpath Construction Causes Path Duplication and Sorting Errors

**Description**
When processing filenames with multiple subpath delimiters, the system incorrectly constructs the subpaths, resulting in duplicated or malformed paths. This leads to inconsistencies and failures in downstream operations that rely on correctly structured subpaths.

**Example Buggy Code**
```python
filename = "p1//p2/p3//n"
subpaths = get_subpaths(filename)
print(subpaths)  # Output: ['p1', 'p1/p1/p2/p3']
```
**Expected Behavior**
- Subpaths should be constructed without duplication, accurately reflecting the intended directory structure.
- Subpaths should be sorted by directory depth and alphabetically to ensure consistent processing.

**Actual Behavior**
- Subpaths are constructed with duplicated path segments, e.g., `['p1', 'p1/p1/p2/p3']` instead of `['p1', 'p1/p2/p3']`.
- Attempting to sort subpaths results in errors due to missing sorting functionality.

This issue affects the reliability of subpath handling within the addurls plugin, causing related tests to fail and potentially leading to incorrect data organization.

[/ISSUE]
{
    "test_formatter": "PASSED",
    "test_formatter_lower_case": "PASSED",
    "test_formatter_no_idx_map": "PASSED",
    "test_formatter_no_mapping_arg": "PASSED",
    "test_formatter_placeholder_with_spaces": "PASSED",
    "test_formatter_placeholder_nonpermitted_chars": "PASSED",
    "test_formatter_missing_arg": "PASSED",
    "test_repformatter": "PASSED",
    "test_clean_meta_args": "PASSED",
    "test_get_subpaths": "PASSED",
    "test_sort_paths": "PASSED",
    "test_is_legal_metafield": "PASSED",
    "test_filter_legal_metafield": "PASSED",
    "test_fmt_to_name": "PASSED",
    "test_get_file_parts": "PASSED",
    "test_get_url_parts": "PASSED",
    "test_extract": "PASSED",
    "test_extract_disable_autometa": "PASSED",
    "test_extract_exclude_autometa_regexp": "PASSED",
    "test_extract_csv_json_equal": "PASSED",
    "test_extract_wrong_input_type": "PASSED",
    "test_addurls_nonannex_repo": "FAILED",
    "test_addurls_dry_run": "FAILED",
    "TestAddurls.test_addurls": "FAILED",
    "TestAddurls.test_addurls_create_newdataset": "FAILED",
    "TestAddurls.test_addurls_subdataset": "FAILED",
    "TestAddurls.test_addurls_repindex": "FAILED",
    "TestAddurls.test_addurls_url_parts": "FAILED",
    "TestAddurls.test_addurls_url_filename": "FAILED",
    "TestAddurls.test_addurls_url_filename_fail": "FAILED",
    "TestAddurls.test_addurls_metafail": "FAILED",
    "TestAddurls.test_addurls_dropped_urls": "FAILED",
    "TestAddurls.test_addurls_version": "FAILED",
    "TestAddurls.test_addurls_deeper": "FAILED"
}
diff --git a/datalad/plugin/addurls.py b/datalad/plugin/addurls.py
index 23100e40c..a98ebc00a 100644
--- a/datalad/plugin/addurls.py
+++ b/datalad/plugin/addurls.py
@@ -176,7 +176,7 @@ def get_subpaths(filename):
 
     spaths = []
     for part in filename.split("//")[:-1]:
-        path = os.path.join(*(spaths + [part]))
+        path = os.path.join(spaths[-1], part) if spaths else part
         spaths.append(path)
     return filename.replace("//", os.path.sep), spaths
 
@@ -373,6 +373,24 @@ def add_extra_filename_values(filename_format, rows, urls, dry_run):
                          "Finished requesting file names")
 
 
+def sort_paths(paths):
+    """Sort `paths` by directory level and then alphabetically.
+
+    Parameters
+    ----------
+    paths : iterable of str
+
+    Returns
+    -------
+    Generator of sorted paths.
+    """
+    def level_and_name(p):
+        return p.count(os.path.sep), p
+
+    for path in sorted(paths, key=level_and_name):
+        yield path
+
+
 def extract(stream, input_type, url_format="{0}", filename_format="{1}",
             exclude_autometa=None, meta=None,
             dry_run=False, missing_value=None):
@@ -440,7 +458,7 @@ def extract(stream, input_type, url_format="{0}", filename_format="{1}",
         RepFormatter(colidx_to_name, missing_value).format,
         filename_format)
     subpaths = _format_filenames(format_filename, rows_with_url, infos)
-    return infos, subpaths
+    return infos, list(sort_paths(subpaths))
 
 
 @with_result_progress("Adding URLs")

{"datalad/plugin/addurls.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections import Mapping\nfrom functools import partial\nimport logging\nimport os\nimport re\nimport string\n\nfrom six import string_types\nfrom six.moves.urllib.parse import urlparse\n\nfrom datalad.dochelpers import exc_str\nfrom datalad.log import log_progress, with_result_progress\nfrom datalad.interface.base import Interface\nfrom datalad.interface.base import build_doc\nfrom datalad.interface.results import annexjson2result, get_status_dict\nfrom datalad.interface.common_opts import nosave_opt\nfrom datalad.support.exceptions import AnnexBatchCommandError\nfrom datalad.support.network import get_url_filename\nfrom datalad.support.path import split_ext\nfrom datalad.support.s3 import get_versioned_url\nfrom datalad.utils import (\n    assure_list,\n    unlink,\n)\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name=None, missing_value=None):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, string_types):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Process metadata arguments.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    A dict mapping field names to values.\n    \"\"\"\n    results = {}\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n        results[field] = value\n    return results\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> from datalad.plugin.addurls import get_subpaths\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(*(spaths + [part]))\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    return bool(re.match(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\", name))\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n\n    Note: This is like `filter(is_legal_metafield, fields)` but the\n    dropped values are logged.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\ndef _read(stream, input_type):\n    if input_type == \"csv\":\n        import csv\n        csvrows = csv.reader(stream)\n        headers = next(csvrows)\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        rows = json.load(stream)\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\"input_type must be 'csv', 'json', or 'ext'\")\n    return rows, idx_map\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        filename = format_fn(row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_file_parts(filename, prefix=\"name\"):\n    \"\"\"Assign a name to various parts of a file.\n\n    Parameters\n    ----------\n    filename : str\n        A file name (no leading path is permitted).\n    prefix : str\n        Prefix to prepend to the key names.\n\n    Returns\n    -------\n    A dict mapping each part to a value.\n    \"\"\"\n    root, ext = split_ext(filename)\n    root_py, ext_py = os.path.splitext(filename)\n\n    return {prefix: filename,\n            prefix + \"_root\": root,\n            prefix + \"_ext\": ext,\n            prefix + \"_root_py\": root_py,\n            prefix + \"_ext_py\": ext_py}\n\n\ndef get_url_parts(url):\n    \"\"\"Assign a name to various parts of the URL.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys `_url_hostname` and, for a path with N+1 parts,\n    '_url0' through '_urlN' .  There is also a `_url_basename` key for\n    the rightmost part of the path.\n    \"\"\"\n    parsed = urlparse(url)\n    if not parsed.netloc:\n        return {}\n\n    names = {\"_url_hostname\": parsed.netloc}\n\n    path = parsed.path.strip(\"/\")\n    if not path:\n        return names\n\n    url_parts = path.split(\"/\")\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    basename = url_parts[-1]\n    names[\"_url_basename\"] = basename\n    names.update(get_file_parts(basename, prefix=\"_url_basename\"))\n    return names\n\n\ndef add_extra_filename_values(filename_format, rows, urls, dry_run):\n    \"\"\"Extend `rows` with values for special formatting fields.\n    \"\"\"\n    file_fields = list(get_fmt_names(filename_format))\n    if any(i.startswith(\"_url\") for i in file_fields):\n        for row, url in zip(rows, urls):\n            row.update(get_url_parts(url))\n\n    if any(i.startswith(\"_url_filename\") for i in file_fields):\n        if dry_run:  # Don't waste time making requests.\n            dummy = get_file_parts(\"BASE.EXT\", \"_url_filename\")\n            for idx, row in enumerate(rows):\n                row.update(\n                    {k: v + str(idx) for k, v in dummy.items()})\n        else:\n            num_urls = len(urls)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Requesting file names for %d URLs\", num_urls,\n                         label=\"Requesting names\", total=num_urls,\n                         unit=\" Files\")\n            for row, url in zip(rows, urls):\n                # If we run into any issues here, we're just going to raise an\n                # exception and then abort inside dlplugin.  It'd be good to\n                # disentangle this from `extract` so that we could yield an\n                # individual error, drop the row, and keep going.\n                filename = get_url_filename(url)\n                if filename:\n                    row.update(get_file_parts(filename, \"_url_filename\"))\n                else:\n                    raise ValueError(\n                        \"{} does not contain a filename\".format(url))\n                log_progress(lgr.info, \"addurls_requestnames\",\n                             \"%s returned for %s\", url, filename,\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Finished requesting file names\")\n\n\ndef extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item is a set that contains all the\n    subdataset paths.\n    \"\"\"\n    meta = assure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n\n    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        url = format_url(row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        meta_args = clean_meta_args(fmt(row) for fmt in formats_meta)\n        infos.append({\"url\": url, \"meta_args\": meta_args})\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, subpaths\n\n\n@with_result_progress(\"Adding URLs\")\ndef add_urls(rows, ifexists=None, options=None):\n    \"\"\"Call `git annex addurl` using information in `rows`.\n    \"\"\"\n    for row in rows:\n        filename_abs = row[\"filename_abs\"]\n        ds, filename = row[\"ds\"], row[\"ds_filename\"]\n        lgr.debug(\"Adding metadata to %s in %s\", filename, ds.path)\n\n        if os.path.exists(filename_abs) or os.path.islink(filename_abs):\n            if ifexists == \"skip\":\n                yield get_status_dict(action=\"addurls\",\n                                      ds=ds,\n                                      type=\"file\",\n                                      path=filename_abs,\n                                      status=\"notneeded\")\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", filename_abs)\n                unlink(filename_abs)\n            else:\n                lgr.debug(\"File %s already exists\", filename_abs)\n\n        try:\n            out_json = ds.repo.add_url_to_file(filename, row[\"url\"],\n                                               batch=True, options=options)\n        except AnnexBatchCommandError as exc:\n            yield get_status_dict(action=\"addurls\",\n                                  ds=ds,\n                                  type=\"file\",\n                                  path=filename_abs,\n                                  message=exc_str(exc),\n                                  status=\"error\")\n            continue\n\n        # In the case of an error, the json object has file=None.\n        if out_json[\"file\"] is None:\n            out_json[\"file\"] = filename_abs\n        yield annexjson2result(out_json, ds, action=\"addurls\",\n                               type=\"file\", logger=lgr)\n\n\n@with_result_progress(\"Adding metadata\")\ndef add_meta(rows):\n    \"\"\"Call `git annex metadata --set` using information in `rows`.\n    \"\"\"\n    from mock import patch\n\n    for row in rows:\n        ds, filename = row[\"ds\"], row[\"ds_filename\"]\n\n        with patch.object(ds.repo, \"always_commit\", False):\n            lgr.debug(\"Adding metadata to %s in %s\", filename, ds.path)\n            for a in ds.repo.set_metadata(filename, add=row[\"meta_args\"]):\n                res = annexjson2result(a, ds, type=\"file\", logger=lgr)\n                # Don't show all added metadata for the file because that\n                # could quickly flood the output.\n                del res[\"message\"]\n                yield res\n\n\n@build_doc\nclass Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a CSV or\n    properties for JSON) are available as placeholders.  If `URL-FILE` is a CSV\n    file, a positional index can also be used (i.e., \"{0}\" for the first\n    column).  Note that a placeholder cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a CSV file (with a header as the first row) or a JSON\n            file (structured as a list of objects with string values).\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded.  The file name may\n            contain directories.  The separator \"//\" can be used to indicate\n            that the left-side directory should be created as a new subdataset.\n            See the 'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file or a JSON\n            file.  The default value, \"ext\", means to consider `URL-FILE` as a\n            JSON file if it ends with \".json\".  Otherwise, treat it as a CSV\n            file.\"\"\",\n            constraints=EnsureChoice(\"ext\", \"csv\", \"json\")),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            metavar=\"ACTION\",\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureNone() | EnsureChoice(\"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on URLs for AWS S3 buckets.\"\"\"),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.interface.results import get_status_dict\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        dataset = require_dataset(dataset, check_installed=False)\n        if dataset.repo and not isinstance(dataset.repo, AnnexRepo):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=\"not an annex repo\")\n            return\n\n        if input_type == \"ext\":\n            extension = os.path.splitext(url_file)[1]\n            input_type = \"json\" if extension == \".json\" else \"csv\"\n\n        with open(url_file) as fd:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield get_status_dict(action=\"addurls\",\n                                      ds=dataset,\n                                      status=\"error\",\n                                      message=exc_str(exc))\n                return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"error\",\n                                  message=(\"There are file name collisions; \"\n                                           \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would download %s to %s\",\n                         row[\"url\"],\n                         os.path.join(dataset.path, row[\"filename\"]))\n                lgr.info(\"Metadata: %s\",\n                         sorted(u\"{}={}\".format(k, v)\n                                for k, v in row[\"meta_args\"].items()))\n            yield get_status_dict(action=\"addurls\",\n                                  ds=dataset,\n                                  status=\"ok\",\n                                  message=\"dry-run finished\")\n            return\n\n        if not dataset.repo:\n            # Populate a new dataset with the URLs.\n            for r in dataset.create(result_xfm=None, return_type='generator',\n                                    save=save):\n                yield r\n\n        annex_options = [\"--fast\"] if fast else []\n\n        for spath in subpaths:\n            if os.path.exists(os.path.join(dataset.path, spath)):\n                lgr.warning(\n                    \"Not creating subdataset at existing path: %s\",\n                    spath)\n            else:\n                for r in dataset.create(spath, result_xfm=None,\n                                        return_type='generator', save=save):\n                    yield r\n\n        for row in rows:\n            # Add additional information that we'll need for various\n            # operations.\n            filename_abs = os.path.join(dataset.path, row[\"filename\"])\n            if row[\"subpath\"]:\n                ds_current = Dataset(os.path.join(dataset.path,\n                                                  row[\"subpath\"]))\n                ds_filename = os.path.relpath(filename_abs, ds_current.path)\n            else:\n                ds_current = dataset\n                ds_filename = row[\"filename\"]\n            row.update({\"filename_abs\": filename_abs,\n                        \"ds\": ds_current,\n                        \"ds_filename\": ds_filename})\n\n        if version_urls:\n            num_urls = len(rows)\n            log_progress(lgr.info, \"addurls_versionurls\",\n                         \"Versioning %d URLs\", num_urls,\n                         label=\"Versioning URLs\",\n                         total=num_urls, unit=\" URLs\")\n            for row in rows:\n                url = row[\"url\"]\n                try:\n                    row[\"url\"] = get_versioned_url(url)\n                except (ValueError, NotImplementedError) as exc:\n                    # We don't expect this to happen because get_versioned_url\n                    # should return the original URL if it isn't an S3 bucket.\n                    # It only raises exceptions if it doesn't know how to\n                    # handle the scheme for what looks like an S3 bucket.\n                    lgr.warning(\"error getting version of %s: %s\",\n                                row[\"url\"], exc_str(exc))\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioned result for %s: %s\", url, row[\"url\"],\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n        files_to_add = set()\n        for r in add_urls(rows, ifexists=ifexists, options=annex_options):\n            if r[\"status\"] == \"ok\":\n                files_to_add.add(r[\"path\"])\n            yield r\n\n            msg = message or \"\"\"\\\n[DATALAD] add files from URLs\n\nurl_file='{}'\nurl_format='{}'\nfilename_format='{}'\"\"\".format(url_file, url_format, filename_format)\n\n        if files_to_add:\n            for r in dataset.add(files_to_add, save=False):\n                yield r\n\n            meta_rows = [r for r in rows if r[\"filename_abs\"] in files_to_add]\n            for r in add_meta(meta_rows):\n                yield r\n\n            # Save here rather than the add call above to trigger a metadata\n            # commit on the git-annex branch.\n            if save:\n                for r in dataset.save(message=msg, recursive=True):\n                    yield r\n\n\n__datalad_plugin__ = Addurls\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
### Incorrect Subpath Construction Causes Path Duplication and Sorting Errors

**Description**
When processing filenames with multiple subpath delimiters, the system incorrectly constructs the subpaths, resulting in duplicated or malformed paths. This leads to inconsistencies and failures in downstream operations that rely on correctly structured subpaths.

**Example Buggy Code**
```python
filename = "p1//p2/p3//n"
subpaths = get_subpaths(filename)
print(subpaths)  # Output: ['p1', 'p1/p1/p2/p3']
```
**Expected Behavior**
- Subpaths should be constructed without duplication, accurately reflecting the intended directory structure.
- Subpaths should be sorted by directory depth and alphabetically to ensure consistent processing.

**Actual Behavior**
- Subpaths are constructed with duplicated path segments, e.g., `['p1', 'p1/p1/p2/p3']` instead of `['p1', 'p1/p2/p3']`.
- Attempting to sort subpaths results in errors due to missing sorting functionality.

This issue affects the reliability of subpath handling within the addurls plugin, causing related tests to fail and potentially leading to incorrect data organization.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections import Mapping
from functools import partial
import logging
import os
import re
import string

from six import string_types
from six.moves.urllib.parse import urlparse

from datalad.dochelpers import exc_str
from datalad.log import log_progress, with_result_progress
from datalad.interface.base import Interface
from datalad.interface.base import build_doc
from datalad.interface.results import annexjson2result, get_status_dict
from datalad.interface.common_opts import nosave_opt
from datalad.support.exceptions import AnnexBatchCommandError
from datalad.support.network import get_url_filename
from datalad.support.path import split_ext
from datalad.support.s3 import get_versioned_url
from datalad.utils import (
    assure_list,
    unlink,
)

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    missing : str, optional
        When column lookup results in an empty string, use this value in
        its place.
    """

    def __init__(self, idx_to_name=None, missing_value=None):
        self.idx_to_name = idx_to_name or {}
        self.missing = missing_value

    def format(self, format_string, *args, **kwargs):
        if not isinstance(args[0], Mapping):
            raise ValueError("First positional argument should be mapping")
        return super(Formatter, self).format(format_string, *args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]

        name = key
        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            name = self.idx_to_name[key_int]

        try:
            value = data[name]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)

        if self.missing is not None and isinstance(value, string_types):
            return value or self.missing
        return value

    def convert_field(self, value, conversion):
        if conversion == 'l':
            return str(value).lower()
        return super(Formatter, self).convert_field(value, conversion)


class RepFormatter(Formatter):
    """Extend Formatter to support a {_repindex} placeholder.
    """

    def __init__(self, *args, **kwargs):
        super(RepFormatter, self).__init__(*args, **kwargs)
        self.repeats = {}
        self.repindex = 0

    def format(self, *args, **kwargs):
        self.repindex = 0
        result = super(RepFormatter, self).format(*args, **kwargs)
        if result in self.repeats:
            self.repindex = self.repeats[result] + 1
            self.repeats[result] = self.repindex
            result = super(RepFormatter, self).format(*args, **kwargs)
        else:
            self.repeats[result] = 0
        return result

    def get_value(self, key, args, kwargs):
        args[0]["_repindex"] = self.repindex
        return super(RepFormatter, self).get_value(key, args, kwargs)


def clean_meta_args(args):
    """Process metadata arguments.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    A dict mapping field names to values.
    """
    results = {}
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            raise ValueError("meta argument isn't in 'field=value' format")

        if not value:
            # The `url_file` may have an empty value.
            continue
        results[field] = value
    return results


def get_subpaths(filename):
    """Convert "//" marker in `filename` to a list of subpaths.

    >>> from datalad.plugin.addurls import get_subpaths
    >>> get_subpaths("p1/p2//p3/p4//file")
    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])

    Note: With Python 3, the subpaths could be generated with

        itertools.accumulate(filename.split("//")[:-1], os.path.join)

    Parameters
    ----------
    filename : str
        File name with "//" marking subpaths.

    Returns
    -------
    A tuple of the filename with any "//" collapsed to a single
    separator and a list of subpaths (str).
    """
    if "//" not in filename:
        return filename, []

    spaths = []
    for part in filename.split("//")[:-1]:
        path = os.path.join(*(spaths + [part]))
        spaths.append(path)
    return filename.replace("//", os.path.sep), spaths


def is_legal_metafield(name):
    """Test whether `name` is a valid metadata field.

    The set of permitted characters is taken from git-annex's
    MetaData.hs:legalField.
    """
    return bool(re.match(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z", name))


def filter_legal_metafield(fields):
    """Remove illegal names from `fields`.

    Note: This is like `filter(is_legal_metafield, fields)` but the
    dropped values are logged.
    """
    legal = []
    for field in fields:
        if is_legal_metafield(field):
            legal.append(field)
        else:
            lgr.debug("%s is not a valid metadata field name; dropping",
                      field)
    return legal


def get_fmt_names(format_string):
    """Yield field names in `format_string`.
    """
    for _, name, _, _ in string.Formatter().parse(format_string):
        if name:
            yield name


def fmt_to_name(format_string, num_to_name):
    """Try to map a format string to a single name.

    Parameters
    ----------
    format_string : string
    num_to_name : dict
        A dictionary that maps from an integer to a column name.  This
        enables mapping the format string to an integer to a name.

    Returns
    -------
    A placeholder name if `format_string` consists of a single
    placeholder and no other text.  Otherwise, None is returned.
    """
    parsed = list(string.Formatter().parse(format_string))
    if len(parsed) != 1:
        # It's an empty string or there's more than one placeholder.
        return
    if parsed[0][0]:
        # Format string contains text before the placeholder.
        return

    name = parsed[0][1]
    if not name:
        # The field name is empty.
        return

    try:
        return num_to_name[int(name)]
    except (KeyError, ValueError):
        return name


def _read(stream, input_type):
    if input_type == "csv":
        import csv
        csvrows = csv.reader(stream)
        headers = next(csvrows)
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        idx_map = dict(enumerate(headers))
        rows = [dict(zip(headers, r)) for r in csvrows]
    elif input_type == "json":
        import json
        rows = json.load(stream)
        # For json input, we do not support indexing by position,
        # only names.
        idx_map = {}
    else:
        raise ValueError("input_type must be 'csv', 'json', or 'ext'")
    return rows, idx_map


def _format_filenames(format_fn, rows, row_infos):
    subpaths = set()
    for row, info in zip(rows, row_infos):
        filename = format_fn(row)
        filename, spaths = get_subpaths(filename)
        subpaths |= set(spaths)
        info["filename"] = filename
        info["subpath"] = spaths[-1] if spaths else None
    return subpaths


def get_file_parts(filename, prefix="name"):
    """Assign a name to various parts of a file.

    Parameters
    ----------
    filename : str
        A file name (no leading path is permitted).
    prefix : str
        Prefix to prepend to the key names.

    Returns
    -------
    A dict mapping each part to a value.
    """
    root, ext = split_ext(filename)
    root_py, ext_py = os.path.splitext(filename)

    return {prefix: filename,
            prefix + "_root": root,
            prefix + "_ext": ext,
            prefix + "_root_py": root_py,
            prefix + "_ext_py": ext_py}


def get_url_parts(url):
    """Assign a name to various parts of the URL.

    Parameters
    ----------
    url : str

    Returns
    -------
    A dict with keys `_url_hostname` and, for a path with N+1 parts,
    '_url0' through '_urlN' .  There is also a `_url_basename` key for
    the rightmost part of the path.
    """
    parsed = urlparse(url)
    if not parsed.netloc:
        return {}

    names = {"_url_hostname": parsed.netloc}

    path = parsed.path.strip("/")
    if not path:
        return names

    url_parts = path.split("/")
    for pidx, part in enumerate(url_parts):
        names["_url{}".format(pidx)] = part
    basename = url_parts[-1]
    names["_url_basename"] = basename
    names.update(get_file_parts(basename, prefix="_url_basename"))
    return names


def add_extra_filename_values(filename_format, rows, urls, dry_run):
    """Extend `rows` with values for special formatting fields.
    """
    file_fields = list(get_fmt_names(filename_format))
    if any(i.startswith("_url") for i in file_fields):
        for row, url in zip(rows, urls):
            row.update(get_url_parts(url))

    if any(i.startswith("_url_filename") for i in file_fields):
        if dry_run:  # Don't waste time making requests.
            dummy = get_file_parts("BASE.EXT", "_url_filename")
            for idx, row in enumerate(rows):
                row.update(
                    {k: v + str(idx) for k, v in dummy.items()})
        else:
            num_urls = len(urls)
            log_progress(lgr.info, "addurls_requestnames",
                         "Requesting file names for %d URLs", num_urls,
                         label="Requesting names", total=num_urls,
                         unit=" Files")
            for row, url in zip(rows, urls):
                # If we run into any issues here, we're just going to raise an
                # exception and then abort inside dlplugin.  It'd be good to
                # disentangle this from `extract` so that we could yield an
                # individual error, drop the row, and keep going.
                filename = get_url_filename(url)
                if filename:
                    row.update(get_file_parts(filename, "_url_filename"))
                else:
                    raise ValueError(
                        "{} does not contain a filename".format(url))
                log_progress(lgr.info, "addurls_requestnames",
                             "%s returned for %s", url, filename,
                             update=1, increment=True)
            log_progress(lgr.info, "addurls_requestnames",
                         "Finished requesting file names")


def extract(stream, input_type, url_format="{0}", filename_format="{1}",
            exclude_autometa=None, meta=None,
            dry_run=False, missing_value=None):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.
    input_type : {'csv', 'json'}

    All other parameters match those described in `AddUrls`.

    Returns
    -------
    A tuple where the first item is a list with a dict of extracted information
    for each row in `stream` and the second item is a set that contains all the
    subdataset paths.
    """
    meta = assure_list(meta)

    rows, colidx_to_name = _read(stream, input_type)

    fmt = Formatter(colidx_to_name, missing_value)  # For URL and meta
    format_url = partial(fmt.format, url_format)

    auto_meta_args = []
    if exclude_autometa not in ["*", ""]:
        urlcol = fmt_to_name(url_format, colidx_to_name)
        # TODO: Try to normalize invalid fields, checking for any
        # collisions.
        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)
        if exclude_autometa:
            metacols = (c for c in metacols
                        if not re.search(exclude_autometa, c))
        metacols = filter_legal_metafield(metacols)
        auto_meta_args = [c + "=" + "{" + c + "}" for c in metacols]

    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]

    rows_with_url = []
    infos = []
    for row in rows:
        url = format_url(row)
        if not url or url == missing_value:
            continue  # pragma: no cover, peephole optimization
        rows_with_url.append(row)
        meta_args = clean_meta_args(fmt(row) for fmt in formats_meta)
        infos.append({"url": url, "meta_args": meta_args})

    n_dropped = len(rows) - len(rows_with_url)
    if n_dropped:
        lgr.warning("Dropped %d row(s) that had an empty URL", n_dropped)

    # Format the filename in a second pass so that we can provide
    # information about the formatted URLs.
    add_extra_filename_values(filename_format, rows_with_url,
                              [i["url"] for i in infos],
                              dry_run)

    # For the file name, we allow the _repindex special key.
    format_filename = partial(
        RepFormatter(colidx_to_name, missing_value).format,
        filename_format)
    subpaths = _format_filenames(format_filename, rows_with_url, infos)
    return infos, subpaths


@with_result_progress("Adding URLs")
def add_urls(rows, ifexists=None, options=None):
    """Call `git annex addurl` using information in `rows`.
    """
    for row in rows:
        filename_abs = row["filename_abs"]
        ds, filename = row["ds"], row["ds_filename"]
        lgr.debug("Adding metadata to %s in %s", filename, ds.path)

        if os.path.exists(filename_abs) or os.path.islink(filename_abs):
            if ifexists == "skip":
                yield get_status_dict(action="addurls",
                                      ds=ds,
                                      type="file",
                                      path=filename_abs,
                                      status="notneeded")
                continue
            elif ifexists == "overwrite":
                lgr.debug("Removing %s", filename_abs)
                unlink(filename_abs)
            else:
                lgr.debug("File %s already exists", filename_abs)

        try:
            out_json = ds.repo.add_url_to_file(filename, row["url"],
                                               batch=True, options=options)
        except AnnexBatchCommandError as exc:
            yield get_status_dict(action="addurls",
                                  ds=ds,
                                  type="file",
                                  path=filename_abs,
                                  message=exc_str(exc),
                                  status="error")
            continue

        # In the case of an error, the json object has file=None.
        if out_json["file"] is None:
            out_json["file"] = filename_abs
        yield annexjson2result(out_json, ds, action="addurls",
                               type="file", logger=lgr)


@with_result_progress("Adding metadata")
def add_meta(rows):
    """Call `git annex metadata --set` using information in `rows`.
    """
    from mock import patch

    for row in rows:
        ds, filename = row["ds"], row["ds_filename"]

        with patch.object(ds.repo, "always_commit", False):
            lgr.debug("Adding metadata to %s in %s", filename, ds.path)
            for a in ds.repo.set_metadata(filename, add=row["meta_args"]):
                res = annexjson2result(a, ds, type="file", logger=lgr)
                # Don't show all added metadata for the file because that
                # could quickly flood the output.
                del res["message"]
                yield res


@build_doc
class Addurls(Interface):
    """Create and update a dataset from a list of URLs.

    *Format specification*

    Several arguments take format strings.  These are similar to normal Python
    format strings where the names from `URL-FILE` (column names for a CSV or
    properties for JSON) are available as placeholders.  If `URL-FILE` is a CSV
    file, a positional index can also be used (i.e., "{0}" for the first
    column).  Note that a placeholder cannot contain a ':' or '!'.

    In addition, the `FILENAME-FORMAT` arguments has a few special
    placeholders.

      - _repindex

        The constructed file names must be unique across all fields rows.  To
        avoid collisions, the special placeholder "_repindex" can be added to
        the formatter.  Its value will start at 0 and increment every time a
        file name repeats.

      - _url_hostname, _urlN, _url_basename*

        Various parts of the formatted URL are available.  Take
        "http://datalad.org/asciicast/seamless_nested_repos.sh" as an example.

        "datalad.org" is stored as "_url_hostname".  Components of the URL's
        path can be referenced as "_urlN".  "_url0" and "_url1" would map to
        "asciicast" and "seamless_nested_repos.sh", respectively.  The final
        part of the path is also available as "_url_basename".

        This name is broken down further.  "_url_basename_root" and
        "_url_basename_ext" provide access to the root name and extension.
        These values are similar to the result of os.path.splitext, but, in the
        case of multiple periods, the extension is identified using the same
        length heuristic that git-annex uses.  As a result, the extension of
        "file.tar.gz" would be ".tar.gz", not ".gz".  In addition, the fields
        "_url_basename_root_py" and "_url_basename_ext_py" provide access to
        the result of os.path.splitext.

      - _url_filename*

        These are similar to _url_basename* fields, but they are obtained with
        a server request.  This is useful if the file name is set in the
        Content-Disposition header.


    *Examples*

    Consider a file "avatars.csv" that contains::

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and 'ext'
    fields, we could run::

      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'

    The `-d avatar_ds` is used to create a new dataset in "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new subdataset in an
    "avatars" subdirectory, we could use "//" in the `FILENAME-FORMAT`
    argument::

      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'

    .. note::

       For users familiar with 'git annex addurl': A large part of this
       plugin's functionality can be viewed as transforming data from
       `URL-FILE` into a "url filename" format that fed to 'git annex addurl
       --batch --with-files'.
    """

    from datalad.distribution.dataset import datasetmethod
    from datalad.interface.utils import eval_results
    from datalad.distribution.dataset import EnsureDataset
    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr
    from datalad.support.param import Parameter

    _params_ = dict(
        dataset=Parameter(
            args=("-d", "--dataset"),
            doc="""Add the URLs to this dataset (or possibly subdatasets of
            this dataset).  An empty or non-existent directory is passed to
            create a new dataset.  New subdatasets can be specified with
            `FILENAME-FORMAT`.""",
            constraints=EnsureDataset() | EnsureNone()),
        urlfile=Parameter(
            args=("urlfile",),
            metavar="URL-FILE",
            doc="""A file that contains URLs or information that can be used to
            construct URLs.  Depending on the value of --input-type, this
            should be a CSV file (with a header as the first row) or a JSON
            file (structured as a list of objects with string values)."""),
        urlformat=Parameter(
            args=("urlformat",),
            metavar="URL-FORMAT",
            doc="""A format string that specifies the URL for each entry.  See
            the 'Format Specification' section above."""),
        filenameformat=Parameter(
            args=("filenameformat",),
            metavar="FILENAME-FORMAT",
            doc="""Like `URL-FORMAT`, but this format string specifies the file
            to which the URL's content will be downloaded.  The file name may
            contain directories.  The separator "//" can be used to indicate
            that the left-side directory should be created as a new subdataset.
            See the 'Format Specification' section above."""),
        input_type=Parameter(
            args=("-t", "--input-type"),
            metavar="TYPE",
            doc="""Whether `URL-FILE` should be considered a CSV file or a JSON
            file.  The default value, "ext", means to consider `URL-FILE` as a
            JSON file if it ends with ".json".  Otherwise, treat it as a CSV
            file.""",
            constraints=EnsureChoice("ext", "csv", "json")),
        exclude_autometa=Parameter(
            args=("-x", "--exclude_autometa"),
            metavar="REGEXP",
            doc="""By default, metadata field=value pairs are constructed with
            each column in `URL-FILE`, excluding any single column that is
            specified via `URL-FORMAT`.  This argument can be used to exclude
            columns that match a regular expression.  If set to '*' or an empty
            string, automatic metadata extraction is disabled completely.  This
            argument does not affect metadata set explicitly with --meta."""),
        meta=Parameter(
            args=("-m", "--meta",),
            metavar="FORMAT",
            action="append",
            doc="""A format string that specifies metadata.  It should be
            structured as "<field>=<value>".  As an example, "location={3}"
            would mean that the value for the "location" metadata field should
            be set the value of the fourth column.  This option can be given
            multiple times."""),
        message=Parameter(
            args=("--message",),
            metavar="MESSAGE",
            doc="""Use this message when committing the URL additions.""",
            constraints=EnsureNone() | EnsureStr()),
        dry_run=Parameter(
            args=("-n", "--dry-run"),
            action="store_true",
            doc="""Report which URLs would be downloaded to which files and
            then exit."""),
        fast=Parameter(
            args=("--fast",),
            action="store_true",
            doc="""If True, add the URLs, but don't download their content.
            Underneath, this passes the --fast flag to `git annex addurl`."""),
        ifexists=Parameter(
            args=("--ifexists",),
            metavar="ACTION",
            doc="""What to do if a constructed file name already exists.  The
            default behavior is to proceed with the `git annex addurl`, which
            will fail if the file size has changed.  If set to 'overwrite',
            remove the old file before adding the new one.  If set to 'skip',
            do not add the new file.""",
            constraints=EnsureNone() | EnsureChoice("overwrite", "skip")),
        missing_value=Parameter(
            args=("--missing-value",),
            metavar="VALUE",
            doc="""When an empty string is encountered, use this value
            instead.""",
            constraints=EnsureNone() | EnsureStr()),
        save=nosave_opt,
        version_urls=Parameter(
            args=("--version-urls",),
            action="store_true",
            doc="""Try to add a version ID to the URL. This currently only has
            an effect on URLs for AWS S3 buckets."""),
    )

    @staticmethod
    @datasetmethod(name='addurls')
    @eval_results
    def __call__(dataset, urlfile, urlformat, filenameformat,
                 input_type="ext", exclude_autometa=None, meta=None,
                 message=None, dry_run=False, fast=False, ifexists=None,
                 missing_value=None, save=True, version_urls=False):
        # Temporarily work around gh-2269.
        url_file = urlfile
        url_format, filename_format = urlformat, filenameformat

        from requests.exceptions import RequestException

        from datalad.distribution.dataset import Dataset, require_dataset
        from datalad.interface.results import get_status_dict
        from datalad.support.annexrepo import AnnexRepo

        lgr = logging.getLogger("datalad.plugin.addurls")

        dataset = require_dataset(dataset, check_installed=False)
        if dataset.repo and not isinstance(dataset.repo, AnnexRepo):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message="not an annex repo")
            return

        if input_type == "ext":
            extension = os.path.splitext(url_file)[1]
            input_type = "json" if extension == ".json" else "csv"

        with open(url_file) as fd:
            try:
                rows, subpaths = extract(fd, input_type,
                                         url_format, filename_format,
                                         exclude_autometa, meta,
                                         dry_run,
                                         missing_value)
            except (ValueError, RequestException) as exc:
                yield get_status_dict(action="addurls",
                                      ds=dataset,
                                      status="error",
                                      message=exc_str(exc))
                return

        if len(rows) != len(set(row["filename"] for row in rows)):
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="error",
                                  message=("There are file name collisions; "
                                           "consider using {_repindex}"))
            return

        if dry_run:
            for subpath in subpaths:
                lgr.info("Would create a subdataset at %s", subpath)
            for row in rows:
                lgr.info("Would download %s to %s",
                         row["url"],
                         os.path.join(dataset.path, row["filename"]))
                lgr.info("Metadata: %s",
                         sorted(u"{}={}".format(k, v)
                                for k, v in row["meta_args"].items()))
            yield get_status_dict(action="addurls",
                                  ds=dataset,
                                  status="ok",
                                  message="dry-run finished")
            return

        if not dataset.repo:
            # Populate a new dataset with the URLs.
            for r in dataset.create(result_xfm=None, return_type='generator',
                                    save=save):
                yield r

        annex_options = ["--fast"] if fast else []

        for spath in subpaths:
            if os.path.exists(os.path.join(dataset.path, spath)):
                lgr.warning(
                    "Not creating subdataset at existing path: %s",
                    spath)
            else:
                for r in dataset.create(spath, result_xfm=None,
                                        return_type='generator', save=save):
                    yield r

        for row in rows:
            # Add additional information that we'll need for various
            # operations.
            filename_abs = os.path.join(dataset.path, row["filename"])
            if row["subpath"]:
                ds_current = Dataset(os.path.join(dataset.path,
                                                  row["subpath"]))
                ds_filename = os.path.relpath(filename_abs, ds_current.path)
            else:
                ds_current = dataset
                ds_filename = row["filename"]
            row.update({"filename_abs": filename_abs,
                        "ds": ds_current,
                        "ds_filename": ds_filename})

        if version_urls:
            num_urls = len(rows)
            log_progress(lgr.info, "addurls_versionurls",
                         "Versioning %d URLs", num_urls,
                         label="Versioning URLs",
                         total=num_urls, unit=" URLs")
            for row in rows:
                url = row["url"]
                try:
                    row["url"] = get_versioned_url(url)
                except (ValueError, NotImplementedError) as exc:
                    # We don't expect this to happen because get_versioned_url
                    # should return the original URL if it isn't an S3 bucket.
                    # It only raises exceptions if it doesn't know how to
                    # handle the scheme for what looks like an S3 bucket.
                    lgr.warning("error getting version of %s: %s",
                                row["url"], exc_str(exc))
                log_progress(lgr.info, "addurls_versionurls",
                             "Versioned result for %s: %s", url, row["url"],
                             update=1, increment=True)
            log_progress(lgr.info, "addurls_versionurls", "Finished versioning URLs")

        files_to_add = set()
        for r in add_urls(rows, ifexists=ifexists, options=annex_options):
            if r["status"] == "ok":
                files_to_add.add(r["path"])
            yield r

            msg = message or """\
[DATALAD] add files from URLs

url_file='{}'
url_format='{}'
filename_format='{}'""".format(url_file, url_format, filename_format)

        if files_to_add:
            for r in dataset.add(files_to_add, save=False):
                yield r

            meta_rows = [r for r in rows if r["filename_abs"] in files_to_add]
            for r in add_meta(meta_rows):
                yield r

            # Save here rather than the add call above to trigger a metadata
            # commit on the git-annex branch.
            if save:
                for r in dataset.save(message=msg, recursive=True):
                    yield r


__datalad_plugin__ = Addurls

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user