{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections.abc import Mapping\n\nimport logging\nimport os\nimport re\nimport string\nimport sys\n\nfrom functools import partial\nfrom urllib.parse import urlparse\n\nfrom datalad.support.external_versions import external_versions\nimport datalad.support.path as op\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.dochelpers import exc_str\nfrom datalad.log import log_progress, with_result_progress\nfrom datalad.interface.base import Interface\nfrom datalad.interface.base import build_doc\nfrom datalad.interface.results import annexjson2result, get_status_dict\nfrom datalad.interface.common_opts import (\n    jobs_opt,\n    nosave_opt,\n)\nfrom datalad.support.exceptions import CommandError\nfrom datalad.support.itertools import groupby_sorted\nfrom datalad.support.network import get_url_filename\nfrom datalad.support.path import split_ext\nfrom datalad.support.parallel import (\n    ProducerConsumerProgressLog,\n    no_parentds_in_futures,\n)\nfrom datalad.support.s3 import get_versioned_url\nfrom datalad.utils import (\n    ensure_list,\n    get_suggestions_msg,\n    Path,\n    unlink,\n)\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name=None, missing_value=None):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, str):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Process metadata arguments.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    A dict mapping field names to values.\n    \"\"\"\n    results = {}\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n        results[field] = value\n    return results\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> from datalad.plugin.addurls import get_subpaths\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(spaths[-1], part) if spaths else part\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    return bool(re.match(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\", name))\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n\n    Note: This is like `filter(is_legal_metafield, fields)` but the\n    dropped values are logged.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\nclass AnnexKeyParser(object):\n    \"\"\"Parse a full annex key into subparts.\n\n    The key may have an \"et:\" prefix appended, which signals that the backend's\n    extension state should be toggled.\n\n    See <https://git-annex.branchable.com/internals/key_format/>.\n\n    Parameters\n    ----------\n    format_fn : callable\n        Function that takes a format string and a row and returns the full key.\n    format_string : str\n        Format string for the full key.\n    \"\"\"\n\n    def __init__(self, format_fn, format_string):\n        self.format_fn = format_fn\n        self.format_string = format_string\n        self.regexp = re.compile(r\"(?P<et>et:)?\"\n                                 r\"(?P<backend>[A-Z0-9]+)\"\n                                 r\"(?:-[^-]+)?\"\n                                 r\"--(?P<keyname>[^/\\n]+)\\Z\")\n        self.empty = \"\".join(i[0]\n                             for i in string.Formatter().parse(format_string))\n\n    @staticmethod\n    def _validate_common(backend, key):\n        if backend.endswith(\"E\"):\n            ext = True\n            backend = backend[:-1]\n        else:\n            ext = False\n\n        expected_lengths = {\"MD5\": 32,\n                            \"SHA1\": 40,\n                            \"SHA256\": 64}\n\n        expected_len = expected_lengths.get(backend)\n        if not expected_len:\n            return\n\n        if ext and \".\" in key:\n            key = key.split(\".\", maxsplit=1)[0]\n\n        if len(key) != expected_len:\n            raise ValueError(\"{} key does not have expected length of {}: {}\"\n                             .format(backend, expected_len, key))\n\n        if not re.match(r\"[a-f0-9]+\\Z\", key):\n            raise ValueError(\"{} key has non-hexadecimal character: {}\"\n                             .format(backend, key))\n\n    def parse(self, row):\n        \"\"\"Format the key with the fields in `row` and parse it.\n\n        Returns\n        -------\n        A dictionary with the following keys that match their counterparts in\n        the output of `git annex examinekey --json`: \"key\" (the full annex\n        key), \"backend\", and \"keyname\". If the key had an \"et:\" prefix, there\n        is also a \"target_backend\" key.\n\n        Raises\n        ------\n        ValueError if the formatted value doesn't look like a valid key\n        \"\"\"\n        try:\n            key = self.format_fn(self.format_string, row)\n        except KeyError as exc:\n            lgr.debug(\"Row missing fields for --key: %s\",\n                      exc_str(exc))\n            return {}\n\n        if key == self.empty:\n            lgr.debug(\"All fields in --key's value are empty in row: %s\", row)\n            # We got the same string that'd we get if all the fields were\n            # empty, so this doesn't have a key.\n            return {}\n\n        match = self.regexp.match(key)\n        if match:\n            info = match.groupdict()\n            et = info.pop(\"et\", None)\n            if et:\n                info[\"key\"] = key[3:]  # Drop \"et:\" from full key.\n                backend = info[\"backend\"]\n                if backend.endswith(\"E\"):\n                    info[\"target_backend\"] = backend[:-1]\n                else:\n                    info[\"target_backend\"] = backend + \"E\"\n            else:\n                info[\"key\"] = key\n            self._validate_common(info[\"backend\"], info[\"keyname\"])\n            return info\n        else:\n            raise ValueError(\n                \"Key does not match expected \"\n                \"[et:]<backend>[-[CmsS]NNN]--<key> format: {}\"\n                .format(key))\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\nINPUT_TYPES = [\"ext\", \"csv\", \"tsv\", \"json\"]\n\n\ndef _read(stream, input_type):\n    if input_type in [\"csv\", \"tsv\"]:\n        import csv\n        csvrows = csv.reader(stream,\n                             delimiter=\"\\t\" if input_type == \"tsv\" else \",\")\n        try:\n            headers = next(csvrows)\n        except StopIteration:\n            raise ValueError(\"Failed to read {} rows from {}\"\n                             .format(input_type.upper(), stream))\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        try:\n            rows = json.load(stream)\n        except json.decoder.JSONDecodeError as e:\n            raise ValueError(\n                \"Failed to read JSON from stream {}: {}\"\n                .format(stream, exc_str(e)))\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\n            \"input_type {} is invalid. Known values: {}\"\n            .format(input_type, \", \".join(INPUT_TYPES)))\n    return rows, idx_map\n\n\ndef _get_placeholder_exception(exc, msg_prefix, known):\n    \"\"\"Recast KeyError as a ValueError with close-match suggestions.\n    \"\"\"\n    value = exc.args[0]\n    if isinstance(value, str):\n        sugmsg = get_suggestions_msg(value, known)\n    else:\n        sugmsg = \"Out-of-bounds or unsupported index.\"\n    # Note: Keeping this a KeyError is probably more appropriate but then the\n    # entire message, which KeyError takes as the key, will be rendered with\n    # outer quotes.\n    return ValueError(\"{}: {}{}{}\"\n                      .format(msg_prefix, exc, \". \" if sugmsg else \"\", sugmsg))\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        try:\n            filename = format_fn(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in file name\", row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_file_parts(filename, prefix=\"name\"):\n    \"\"\"Assign a name to various parts of a file.\n\n    Parameters\n    ----------\n    filename : str\n        A file name (no leading path is permitted).\n    prefix : str\n        Prefix to prepend to the key names.\n\n    Returns\n    -------\n    A dict mapping each part to a value.\n    \"\"\"\n    root, ext = split_ext(filename)\n    root_py, ext_py = os.path.splitext(filename)\n\n    return {prefix: filename,\n            prefix + \"_root\": root,\n            prefix + \"_ext\": ext,\n            prefix + \"_root_py\": root_py,\n            prefix + \"_ext_py\": ext_py}\n\n\ndef get_url_parts(url):\n    \"\"\"Assign a name to various parts of the URL.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys `_url_hostname` and, for a path with N+1 parts,\n    '_url0' through '_urlN' .  There is also a `_url_basename` key for\n    the rightmost part of the path.\n    \"\"\"\n    parsed = urlparse(url)\n    if not parsed.netloc:\n        return {}\n\n    names = {\"_url_hostname\": parsed.netloc}\n\n    path = parsed.path.strip(\"/\")\n    if not path:\n        return names\n\n    url_parts = path.split(\"/\")\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    basename = url_parts[-1]\n    names[\"_url_basename\"] = basename\n    names.update(get_file_parts(basename, prefix=\"_url_basename\"))\n    return names\n\n\ndef add_extra_filename_values(filename_format, rows, urls, dry_run):\n    \"\"\"Extend `rows` with values for special formatting fields.\n    \"\"\"\n    file_fields = list(get_fmt_names(filename_format))\n    if any(i.startswith(\"_url\") for i in file_fields):\n        for row, url in zip(rows, urls):\n            row.update(get_url_parts(url))\n\n    if any(i.startswith(\"_url_filename\") for i in file_fields):\n        if dry_run:  # Don't waste time making requests.\n            dummy = get_file_parts(\"BASE.EXT\", \"_url_filename\")\n            for idx, row in enumerate(rows):\n                row.update(\n                    {k: v + str(idx) for k, v in dummy.items()})\n        else:\n            num_urls = len(urls)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Requesting file names for %d URLs\", num_urls,\n                         label=\"Requesting names\", total=num_urls,\n                         unit=\" Files\")\n            for row, url in zip(rows, urls):\n                # If we run into any issues here, we're just going to raise an\n                # exception and then abort inside dlplugin.  It'd be good to\n                # disentangle this from `extract` so that we could yield an\n                # individual error, drop the row, and keep going.\n                filename = get_url_filename(url)\n                if filename:\n                    row.update(get_file_parts(filename, \"_url_filename\"))\n                else:\n                    raise ValueError(\n                        \"{} does not contain a filename\".format(url))\n                log_progress(lgr.info, \"addurls_requestnames\",\n                             \"%s returned for %s\", url, filename,\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Finished requesting file names\")\n\n\ndef sort_paths(paths):\n    \"\"\"Sort `paths` by directory level and then alphabetically.\n\n    Parameters\n    ----------\n    paths : iterable of str\n\n    Returns\n    -------\n    Generator of sorted paths.\n    \"\"\"\n    def level_and_name(p):\n        return p.count(os.path.sep), p\n\n    yield from sorted(paths, key=level_and_name)\n\n\ndef extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None, key=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'tsv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item a list subdataset paths,\n    sorted breadth-first.\n    \"\"\"\n    meta = ensure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n    if not rows:\n        lgr.warning(\"No rows found in %s\", stream)\n        return [], []\n\n    # Formatter for everything but file names\n    fmt = Formatter(colidx_to_name, missing_value)\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    info_fns = []\n    if formats_meta:\n        def set_meta_args(info, row):\n            info[\"meta_args\"] = clean_meta_args(fmt(row)\n                                                for fmt in formats_meta)\n        info_fns.append(set_meta_args)\n    if key:\n        key_parser = AnnexKeyParser(fmt.format, key)\n\n        def set_key(info, row):\n            info[\"key\"] = key_parser.parse(row)\n        info_fns.append(set_key)\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        try:\n            url = format_url(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in URL\", row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        info = {\"url\": url}\n        for fn in info_fns:\n            fn(info, row)\n        infos.append(info)\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, list(sort_paths(subpaths))\n\n\ndef _add_url(row, ds, repo, options=None, drop_after=False):\n    filename_abs = row[\"filename_abs\"]\n    filename = row[\"ds_filename\"]\n    try:\n        out_json = repo.add_url_to_file(filename, row[\"url\"],\n                                        batch=True, options=options)\n    except CommandError as exc:\n        yield get_status_dict(action=\"addurls\",\n                              ds=ds,\n                              type=\"file\",\n                              path=filename_abs,\n                              message=exc_str(exc),\n                              status=\"error\")\n        return\n\n    # In the case of an error, the json object has file=None.\n    if out_json[\"file\"] is None:\n        out_json[\"file\"] = filename_abs\n    res_addurls = annexjson2result(\n        out_json, ds, action=\"addurls\",\n        type=\"file\", logger=lgr)\n    yield res_addurls\n\n    if not res_addurls[\"status\"] == \"ok\":\n        return\n\n    if drop_after and 'annexkey' in res_addurls:\n        # unfortunately .drop has no batched mode, and drop_key ATM would\n        # raise AssertionError if not success, and otherwise return nothing\n        try:\n            repo.drop_key(res_addurls['annexkey'], batch=True)\n            st_kwargs = dict(status=\"ok\")\n        except (AssertionError, CommandError) as exc:\n            st_kwargs = dict(message=exc_str(exc),\n                             status=\"error\")\n        yield get_status_dict(action=\"drop\",\n                              ds=ds,\n                              annexkey=res_addurls['annexkey'],\n                              type=\"file\",\n                              path=filename_abs,\n                              **st_kwargs)\n\n\nclass RegisterUrl(object):\n    \"\"\"Create files (without content) from user-supplied keys and register URLs.\n    \"\"\"\n\n    def __init__(self, ds, repo=None):\n        self.ds = ds\n        self.repo = repo or ds.repo\n        self._err_res = get_status_dict(action=\"addurls\", ds=self.ds,\n                                        type=\"file\", status=\"error\")\n        self.use_pointer = repo.is_managed_branch()\n\n    def examinekey(self, parsed_key, filename, migrate=False):\n        opts = []\n        if migrate:\n            opts.append(\"--migrate-to-backend=\" + parsed_key[\"target_backend\"])\n        opts.extend([\"--filename=\" + filename, parsed_key[\"key\"]])\n        return self.repo.call_annex_records([\"examinekey\"] + opts)[0]\n\n    def fromkey(self, key, filename):\n        return self.repo.call_annex_records(\n            [\"fromkey\", \"--force\", key, filename])[0]\n\n    def registerurl(self, key, url):\n        self.repo.call_annex([\"registerurl\", key, url])\n\n    def _write_pointer(self, row, ek_info):\n        try:\n            fname = Path(row[\"filename_abs\"])\n            fname.parent.mkdir(exist_ok=True, parents=True)\n            fname.write_text(ek_info[\"objectpointer\"])\n        except Exception as exc:\n            message = exc_str(exc)\n            status = \"error\"\n        else:\n            message = \"registered URL\"\n            status = \"ok\"\n        return get_status_dict(action=\"addurls\", ds=self.ds, type=\"file\",\n                               status=status, message=message)\n\n    def __call__(self, row):\n        filename = row[\"ds_filename\"]\n        try:\n            parsed_key = row[\"key\"]\n            migrate = \"target_backend\" in parsed_key\n            use_pointer = self.use_pointer\n            ek_info = None\n            if use_pointer or migrate:\n                ek_info = self.examinekey(parsed_key, filename,\n                                          migrate=migrate)\n                if not ek_info:\n                    yield dict(self._err_res,\n                               path=row[\"filename_abs\"],\n                               message=(\"Failed to get information for %s\",\n                                        parsed_key))\n                    return\n                key = ek_info[\"key\"]\n            else:\n                key = parsed_key[\"key\"]\n\n            self.registerurl(key, row[\"url\"])\n            if use_pointer:\n                res = self._write_pointer(row, ek_info)\n            else:\n                res = annexjson2result(self.fromkey(key, filename),\n                                       self.ds, type=\"file\", logger=lgr)\n                if not res.get(\"message\"):\n                    res[\"message\"] = \"registered URL\"\n        except CommandError as exc:\n            yield dict(self._err_res,\n                       path=row[\"filename_abs\"],\n                       message=exc_str(exc))\n        else:\n            yield res\n\n\n# Note: If any other modules end up needing these batch operations, this should\n# find a new home.\n\n\nclass BatchedRegisterUrl(RegisterUrl):\n    \"\"\"Like `RegisterUrl`, but use batched commands underneath.\n    \"\"\"\n\n    def __init__(self, ds, repo=None):\n        super().__init__(ds, repo)\n        self._batch_commands = {}\n\n    def _batch(self, command, batch_input,\n               output_proc=None, json=False, batch_options=None):\n        bcmd = self._batch_commands.get(command)\n        if not bcmd:\n            repo = self.repo\n            bcmd = repo._batched.get(\n                command, path=repo.path, json=json, output_proc=output_proc,\n                annex_options=batch_options)\n            self._batch_commands[command] = bcmd\n        return bcmd(batch_input)\n\n    def examinekey(self, parsed_key, filename, migrate=False):\n        if migrate:\n            opts = [\"--migrate-to-backend=\" + parsed_key[\"target_backend\"]]\n        else:\n            opts = None\n        return self._batch(\"examinekey\", (parsed_key[\"key\"], filename),\n                           json=True, batch_options=opts)\n\n    def fromkey(self, key, filename):\n        return self._batch(\"fromkey\", (key, filename), json=True,\n                           # --force is needed because the key (usually) does\n                           # not exist in the local repository.\n                           batch_options=[\"--force\"])\n\n    @staticmethod\n    def _ignore(_stdout):\n        return\n\n    def registerurl(self, key, url):\n        self._batch(\"registerurl\", (key, url),\n                    output_proc=self._ignore, json=False)\n\n\ndef _log_filter_addurls(res):\n    return res.get('type') == 'file' and res.get('action') in [\"addurl\", \"addurls\"]\n\n\n@with_result_progress(\"Adding URLs\", log_filter=_log_filter_addurls)\ndef _add_urls(rows, ds, repo, ifexists=None, options=None,\n              drop_after=False, by_key=False):\n    \"\"\"Call `git annex addurl` using information in `rows`.\n    \"\"\"\n    add_url = partial(_add_url, ds=ds, repo=repo,\n                      drop_after=drop_after, options=options)\n    if by_key:\n        # The by_key parameter isn't strictly needed, but it lets us avoid some\n        # setup if --key wasn't specified.\n        if repo.fake_dates_enabled:\n            register_url = RegisterUrl(ds, repo)\n        else:\n            register_url = BatchedRegisterUrl(ds, repo)\n    else:\n        def register_url(*args, **kwargs):\n            raise RuntimeError(\"bug: this should be impossible\")\n\n    add_metadata = {}\n    for row in rows:\n        filename_abs = row[\"filename_abs\"]\n        filename = row[\"ds_filename\"]\n        lgr.debug(\"Adding URLs to %s in %s\", filename, ds.path)\n\n        if os.path.exists(filename_abs) or os.path.islink(filename_abs):\n            if ifexists == \"skip\":\n                yield get_status_dict(action=\"addurls\",\n                                      ds=ds,\n                                      type=\"file\",\n                                      path=filename_abs,\n                                      status=\"notneeded\")\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", filename_abs)\n                unlink(filename_abs)\n            else:\n                lgr.debug(\"File %s already exists\", filename_abs)\n\n        fn = register_url if row.get(\"key\") else add_url\n        all_ok = True\n        for res in fn(row):\n            if res[\"status\"] != \"ok\":\n                all_ok = False\n            yield res\n        if not all_ok:\n            continue\n\n        if row.get(\"meta_args\"):\n            add_metadata[filename] = row[\"meta_args\"]\n\n    if not add_metadata:\n        return\n\n    # For some reason interleaving batched addurl and regular metadata calls\n    # causes multiple commits, so we do need to have it separate.\n    # TODO: figure out with Joey either could be avoided\n    from unittest.mock import patch\n    with patch.object(repo, \"always_commit\", False):\n        for filename, meta in add_metadata.items():\n            lgr.debug(\"Adding metadata to %s in %s\", filename, repo.path)\n            assert not repo.always_commit\n            for a in repo.set_metadata_(filename, add=meta):\n                res = annexjson2result(a, ds, type=\"file\", logger=lgr)\n                if res[\"status\"] == \"ok\":\n                    # Don't show all added metadata for the file because that\n                    # could quickly flood the output.\n                    del res[\"message\"]\n                yield res\n\n\n@build_doc\nclass Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')\n\n\n__datalad_plugin__ = Addurls\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections.abc import Mapping\n\nimport logging\nimport os\nimport re\nimport string\nimport sys\n\nfrom functools import partial\nfrom urllib.parse import urlparse\n\nfrom datalad.support.external_versions import external_versions\nimport datalad.support.path as op\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.dochelpers import exc_str\nfrom datalad.log import log_progress, with_result_progress\nfrom datalad.interface.base import Interface\nfrom datalad.interface.base import build_doc\nfrom datalad.interface.results import annexjson2result, get_status_dict\nfrom datalad.interface.common_opts import (\n    jobs_opt,\n    nosave_opt,\n)\nfrom datalad.support.exceptions import CommandError\nfrom datalad.support.itertools import groupby_sorted\nfrom datalad.support.network import get_url_filename\nfrom datalad.support.path import split_ext\nfrom datalad.support.parallel import (\n    ProducerConsumerProgressLog,\n    no_parentds_in_futures,\n)\nfrom datalad.support.s3 import get_versioned_url\nfrom datalad.utils import (\n    ensure_list,\n    get_suggestions_msg,\n    Path,\n    unlink,\n)\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name=None, missing_value=None):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, str):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Process metadata arguments.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    A dict mapping field names to values.\n    \"\"\"\n    results = {}\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n        results[field] = value\n    return results\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> from datalad.plugin.addurls import get_subpaths\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(spaths[-1], part) if spaths else part\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    return bool(re.match(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\", name))\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n\n    Note: This is like `filter(is_legal_metafield, fields)` but the\n    dropped values are logged.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\nclass AnnexKeyParser(object):\n    \"\"\"Parse a full annex key into subparts.\n\n    The key may have an \"et:\" prefix appended, which signals that the backend's\n    extension state should be toggled.\n\n    See <https://git-annex.branchable.com/internals/key_format/>.\n\n    Parameters\n    ----------\n    format_fn : callable\n        Function that takes a format string and a row and returns the full key.\n    format_string : str\n        Format string for the full key.\n    \"\"\"\n\n    def __init__(self, format_fn, format_string):\n        self.format_fn = format_fn\n        self.format_string = format_string\n        self.regexp = re.compile(r\"(?P<et>et:)?\"\n                                 r\"(?P<backend>[A-Z0-9]+)\"\n                                 r\"(?:-[^-]+)?\"\n                                 r\"--(?P<keyname>[^/\\n]+)\\Z\")\n        self.empty = \"\".join(i[0]\n                             for i in string.Formatter().parse(format_string))\n\n    @staticmethod\n    def _validate_common(backend, key):\n        if backend.endswith(\"E\"):\n            ext = True\n            backend = backend[:-1]\n        else:\n            ext = False\n\n        expected_lengths = {\"MD5\": 32,\n                            \"SHA1\": 40,\n                            \"SHA256\": 64}\n\n        expected_len = expected_lengths.get(backend)\n        if not expected_len:\n            return\n\n        if ext and \".\" in key:\n            key = key.split(\".\", maxsplit=1)[0]\n\n        if len(key) != expected_len:\n            raise ValueError(\"{} key does not have expected length of {}: {}\"\n                             .format(backend, expected_len, key))\n\n        if not re.match(r\"[a-f0-9]+\\Z\", key):\n            raise ValueError(\"{} key has non-hexadecimal character: {}\"\n                             .format(backend, key))\n\n    def parse(self, row):\n        \"\"\"Format the key with the fields in `row` and parse it.\n\n        Returns\n        -------\n        A dictionary with the following keys that match their counterparts in\n        the output of `git annex examinekey --json`: \"key\" (the full annex\n        key), \"backend\", and \"keyname\". If the key had an \"et:\" prefix, there\n        is also a \"target_backend\" key.\n\n        Raises\n        ------\n        ValueError if the formatted value doesn't look like a valid key\n        \"\"\"\n        try:\n            key = self.format_fn(self.format_string, row)\n        except KeyError as exc:\n            lgr.debug(\"Row missing fields for --key: %s\",\n                      exc_str(exc))\n            return {}\n\n        if key == self.empty:\n            lgr.debug(\"All fields in --key's value are empty in row: %s\", row)\n            # We got the same string that'd we get if all the fields were\n            # empty, so this doesn't have a key.\n            return {}\n\n        match = self.regexp.match(key)\n        if match:\n            info = match.groupdict()\n            et = info.pop(\"et\", None)\n            if et:\n                info[\"key\"] = key[3:]  # Drop \"et:\" from full key.\n                backend = info[\"backend\"]\n                if backend.endswith(\"E\"):\n                    info[\"target_backend\"] = backend[:-1]\n                else:\n                    info[\"target_backend\"] = backend + \"E\"\n            else:\n                info[\"key\"] = key\n            self._validate_common(info[\"backend\"], info[\"keyname\"])\n            return info\n        else:\n            raise ValueError(\n                \"Key does not match expected \"\n                \"[et:]<backend>[-[CmsS]NNN]--<key> format: {}\"\n                .format(key))\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\nINPUT_TYPES = [\"ext\", \"csv\", \"tsv\", \"json\"]\n\n\ndef _read(stream, input_type):\n    if input_type in [\"csv\", \"tsv\"]:\n        import csv\n        csvrows = csv.reader(stream,\n                             delimiter=\"\\t\" if input_type == \"tsv\" else \",\")\n        try:\n            headers = next(csvrows)\n        except StopIteration:\n            raise ValueError(\"Failed to read {} rows from {}\"\n                             .format(input_type.upper(), stream))\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        try:\n            rows = json.load(stream)\n        except json.decoder.JSONDecodeError as e:\n            raise ValueError(\n                \"Failed to read JSON from stream {}: {}\"\n                .format(stream, exc_str(e)))\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\n            \"input_type {} is invalid. Known values: {}\"\n            .format(input_type, \", \".join(INPUT_TYPES)))\n    return rows, idx_map\n\n\ndef _read_from_file(fname, input_type):\n    from_stdin = fname == \"-\"\n    if input_type == \"ext\":\n        if from_stdin:\n            input_type = \"json\"\n        else:\n            extension = os.path.splitext(fname)[1]\n            if extension == \".json\":\n                input_type = \"json\"\n            elif extension == \".tsv\":\n                input_type = \"tsv\"\n            else:\n                input_type = \"csv\"\n\n    fd = sys.stdin if from_stdin else open(fname)\n    try:\n        records, colidx_to_name = _read(fd, input_type)\n        if not records:\n            lgr.warning(\"No rows found in %s\", fd)\n    finally:\n        if fd is not sys.stdin:\n            fd.close()\n    return records, colidx_to_name\n\n\ndef _get_placeholder_exception(exc, msg_prefix, known):\n    \"\"\"Recast KeyError as a ValueError with close-match suggestions.\n    \"\"\"\n    value = exc.args[0]\n    if isinstance(value, str):\n        sugmsg = get_suggestions_msg(value, known)\n    else:\n        sugmsg = \"Out-of-bounds or unsupported index.\"\n    # Note: Keeping this a KeyError is probably more appropriate but then the\n    # entire message, which KeyError takes as the key, will be rendered with\n    # outer quotes.\n    return ValueError(\"{}: {}{}{}\"\n                      .format(msg_prefix, exc, \". \" if sugmsg else \"\", sugmsg))\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        try:\n            filename = format_fn(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in file name\", row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_file_parts(filename, prefix=\"name\"):\n    \"\"\"Assign a name to various parts of a file.\n\n    Parameters\n    ----------\n    filename : str\n        A file name (no leading path is permitted).\n    prefix : str\n        Prefix to prepend to the key names.\n\n    Returns\n    -------\n    A dict mapping each part to a value.\n    \"\"\"\n    root, ext = split_ext(filename)\n    root_py, ext_py = os.path.splitext(filename)\n\n    return {prefix: filename,\n            prefix + \"_root\": root,\n            prefix + \"_ext\": ext,\n            prefix + \"_root_py\": root_py,\n            prefix + \"_ext_py\": ext_py}\n\n\ndef get_url_parts(url):\n    \"\"\"Assign a name to various parts of the URL.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys `_url_hostname` and, for a path with N+1 parts,\n    '_url0' through '_urlN' .  There is also a `_url_basename` key for\n    the rightmost part of the path.\n    \"\"\"\n    parsed = urlparse(url)\n    if not parsed.netloc:\n        return {}\n\n    names = {\"_url_hostname\": parsed.netloc}\n\n    path = parsed.path.strip(\"/\")\n    if not path:\n        return names\n\n    url_parts = path.split(\"/\")\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    basename = url_parts[-1]\n    names[\"_url_basename\"] = basename\n    names.update(get_file_parts(basename, prefix=\"_url_basename\"))\n    return names\n\n\ndef add_extra_filename_values(filename_format, rows, urls, dry_run):\n    \"\"\"Extend `rows` with values for special formatting fields.\n    \"\"\"\n    file_fields = list(get_fmt_names(filename_format))\n    if any(i.startswith(\"_url\") for i in file_fields):\n        for row, url in zip(rows, urls):\n            row.update(get_url_parts(url))\n\n    if any(i.startswith(\"_url_filename\") for i in file_fields):\n        if dry_run:  # Don't waste time making requests.\n            dummy = get_file_parts(\"BASE.EXT\", \"_url_filename\")\n            for idx, row in enumerate(rows):\n                row.update(\n                    {k: v + str(idx) for k, v in dummy.items()})\n        else:\n            num_urls = len(urls)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Requesting file names for %d URLs\", num_urls,\n                         label=\"Requesting names\", total=num_urls,\n                         unit=\" Files\")\n            for row, url in zip(rows, urls):\n                # If we run into any issues here, we're just going to raise an\n                # exception and then abort inside dlplugin.  It'd be good to\n                # disentangle this from `extract` so that we could yield an\n                # individual error, drop the row, and keep going.\n                filename = get_url_filename(url)\n                if filename:\n                    row.update(get_file_parts(filename, \"_url_filename\"))\n                else:\n                    raise ValueError(\n                        \"{} does not contain a filename\".format(url))\n                log_progress(lgr.info, \"addurls_requestnames\",\n                             \"%s returned for %s\", url, filename,\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Finished requesting file names\")\n\n\ndef sort_paths(paths):\n    \"\"\"Sort `paths` by directory level and then alphabetically.\n\n    Parameters\n    ----------\n    paths : iterable of str\n\n    Returns\n    -------\n    Generator of sorted paths.\n    \"\"\"\n    def level_and_name(p):\n        return p.count(os.path.sep), p\n\n    yield from sorted(paths, key=level_and_name)\n\n\ndef extract(rows, colidx_to_name=None,\n            url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None, key=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `rows`.\n\n    Parameters\n    ----------\n    rows : list of dict\n    colidx_to_name : dict, optional\n        Mapping from a position index to a column name.\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item a list subdataset paths,\n    sorted breadth-first.\n    \"\"\"\n    meta = ensure_list(meta)\n    colidx_to_name = colidx_to_name or {}\n\n    # Formatter for everything but file names\n    fmt = Formatter(colidx_to_name, missing_value)\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    info_fns = []\n    if formats_meta:\n        def set_meta_args(info, row):\n            info[\"meta_args\"] = clean_meta_args(fmt(row)\n                                                for fmt in formats_meta)\n        info_fns.append(set_meta_args)\n    if key:\n        key_parser = AnnexKeyParser(fmt.format, key)\n\n        def set_key(info, row):\n            info[\"key\"] = key_parser.parse(row)\n        info_fns.append(set_key)\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        try:\n            url = format_url(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in URL\", row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        info = {\"url\": url}\n        for fn in info_fns:\n            fn(info, row)\n        infos.append(info)\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, list(sort_paths(subpaths))\n\n\ndef _add_url(row, ds, repo, options=None, drop_after=False):\n    filename_abs = row[\"filename_abs\"]\n    filename = row[\"ds_filename\"]\n    try:\n        out_json = repo.add_url_to_file(filename, row[\"url\"],\n                                        batch=True, options=options)\n    except CommandError as exc:\n        yield get_status_dict(action=\"addurls\",\n                              ds=ds,\n                              type=\"file\",\n                              path=filename_abs,\n                              message=exc_str(exc),\n                              status=\"error\")\n        return\n\n    # In the case of an error, the json object has file=None.\n    if out_json[\"file\"] is None:\n        out_json[\"file\"] = filename_abs\n    res_addurls = annexjson2result(\n        out_json, ds, action=\"addurls\",\n        type=\"file\", logger=lgr)\n    yield res_addurls\n\n    if not res_addurls[\"status\"] == \"ok\":\n        return\n\n    if drop_after and 'annexkey' in res_addurls:\n        # unfortunately .drop has no batched mode, and drop_key ATM would\n        # raise AssertionError if not success, and otherwise return nothing\n        try:\n            repo.drop_key(res_addurls['annexkey'], batch=True)\n            st_kwargs = dict(status=\"ok\")\n        except (AssertionError, CommandError) as exc:\n            st_kwargs = dict(message=exc_str(exc),\n                             status=\"error\")\n        yield get_status_dict(action=\"drop\",\n                              ds=ds,\n                              annexkey=res_addurls['annexkey'],\n                              type=\"file\",\n                              path=filename_abs,\n                              **st_kwargs)\n\n\nclass RegisterUrl(object):\n    \"\"\"Create files (without content) from user-supplied keys and register URLs.\n    \"\"\"\n\n    def __init__(self, ds, repo=None):\n        self.ds = ds\n        self.repo = repo or ds.repo\n        self._err_res = get_status_dict(action=\"addurls\", ds=self.ds,\n                                        type=\"file\", status=\"error\")\n        self.use_pointer = repo.is_managed_branch()\n\n    def examinekey(self, parsed_key, filename, migrate=False):\n        opts = []\n        if migrate:\n            opts.append(\"--migrate-to-backend=\" + parsed_key[\"target_backend\"])\n        opts.extend([\"--filename=\" + filename, parsed_key[\"key\"]])\n        return self.repo.call_annex_records([\"examinekey\"] + opts)[0]\n\n    def fromkey(self, key, filename):\n        return self.repo.call_annex_records(\n            [\"fromkey\", \"--force\", key, filename])[0]\n\n    def registerurl(self, key, url):\n        self.repo.call_annex([\"registerurl\", key, url])\n\n    def _write_pointer(self, row, ek_info):\n        try:\n            fname = Path(row[\"filename_abs\"])\n            fname.parent.mkdir(exist_ok=True, parents=True)\n            fname.write_text(ek_info[\"objectpointer\"])\n        except Exception as exc:\n            message = exc_str(exc)\n            status = \"error\"\n        else:\n            message = \"registered URL\"\n            status = \"ok\"\n        return get_status_dict(action=\"addurls\", ds=self.ds, type=\"file\",\n                               status=status, message=message)\n\n    def __call__(self, row):\n        filename = row[\"ds_filename\"]\n        try:\n            parsed_key = row[\"key\"]\n            migrate = \"target_backend\" in parsed_key\n            use_pointer = self.use_pointer\n            ek_info = None\n            if use_pointer or migrate:\n                ek_info = self.examinekey(parsed_key, filename,\n                                          migrate=migrate)\n                if not ek_info:\n                    yield dict(self._err_res,\n                               path=row[\"filename_abs\"],\n                               message=(\"Failed to get information for %s\",\n                                        parsed_key))\n                    return\n                key = ek_info[\"key\"]\n            else:\n                key = parsed_key[\"key\"]\n\n            self.registerurl(key, row[\"url\"])\n            if use_pointer:\n                res = self._write_pointer(row, ek_info)\n            else:\n                res = annexjson2result(self.fromkey(key, filename),\n                                       self.ds, type=\"file\", logger=lgr)\n                if not res.get(\"message\"):\n                    res[\"message\"] = \"registered URL\"\n        except CommandError as exc:\n            yield dict(self._err_res,\n                       path=row[\"filename_abs\"],\n                       message=exc_str(exc))\n        else:\n            yield res\n\n\n# Note: If any other modules end up needing these batch operations, this should\n# find a new home.\n\n\nclass BatchedRegisterUrl(RegisterUrl):\n    \"\"\"Like `RegisterUrl`, but use batched commands underneath.\n    \"\"\"\n\n    def __init__(self, ds, repo=None):\n        super().__init__(ds, repo)\n        self._batch_commands = {}\n\n    def _batch(self, command, batch_input,\n               output_proc=None, json=False, batch_options=None):\n        bcmd = self._batch_commands.get(command)\n        if not bcmd:\n            repo = self.repo\n            bcmd = repo._batched.get(\n                command, path=repo.path, json=json, output_proc=output_proc,\n                annex_options=batch_options)\n            self._batch_commands[command] = bcmd\n        return bcmd(batch_input)\n\n    def examinekey(self, parsed_key, filename, migrate=False):\n        if migrate:\n            opts = [\"--migrate-to-backend=\" + parsed_key[\"target_backend\"]]\n        else:\n            opts = None\n        return self._batch(\"examinekey\", (parsed_key[\"key\"], filename),\n                           json=True, batch_options=opts)\n\n    def fromkey(self, key, filename):\n        return self._batch(\"fromkey\", (key, filename), json=True,\n                           # --force is needed because the key (usually) does\n                           # not exist in the local repository.\n                           batch_options=[\"--force\"])\n\n    @staticmethod\n    def _ignore(_stdout):\n        return\n\n    def registerurl(self, key, url):\n        self._batch(\"registerurl\", (key, url),\n                    output_proc=self._ignore, json=False)\n\n\ndef _log_filter_addurls(res):\n    return res.get('type') == 'file' and res.get('action') in [\"addurl\", \"addurls\"]\n\n\n@with_result_progress(\"Adding URLs\", log_filter=_log_filter_addurls)\ndef _add_urls(rows, ds, repo, ifexists=None, options=None,\n              drop_after=False, by_key=False):\n    \"\"\"Call `git annex addurl` using information in `rows`.\n    \"\"\"\n    add_url = partial(_add_url, ds=ds, repo=repo,\n                      drop_after=drop_after, options=options)\n    if by_key:\n        # The by_key parameter isn't strictly needed, but it lets us avoid some\n        # setup if --key wasn't specified.\n        if repo.fake_dates_enabled:\n            register_url = RegisterUrl(ds, repo)\n        else:\n            register_url = BatchedRegisterUrl(ds, repo)\n    else:\n        def register_url(*args, **kwargs):\n            raise RuntimeError(\"bug: this should be impossible\")\n\n    add_metadata = {}\n    for row in rows:\n        filename_abs = row[\"filename_abs\"]\n        filename = row[\"ds_filename\"]\n        lgr.debug(\"Adding URLs to %s in %s\", filename, ds.path)\n\n        if os.path.exists(filename_abs) or os.path.islink(filename_abs):\n            if ifexists == \"skip\":\n                yield get_status_dict(action=\"addurls\",\n                                      ds=ds,\n                                      type=\"file\",\n                                      path=filename_abs,\n                                      status=\"notneeded\")\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", filename_abs)\n                unlink(filename_abs)\n            else:\n                lgr.debug(\"File %s already exists\", filename_abs)\n\n        fn = register_url if row.get(\"key\") else add_url\n        all_ok = True\n        for res in fn(row):\n            if res[\"status\"] != \"ok\":\n                all_ok = False\n            yield res\n        if not all_ok:\n            continue\n\n        if row.get(\"meta_args\"):\n            add_metadata[filename] = row[\"meta_args\"]\n\n    if not add_metadata:\n        return\n\n    # For some reason interleaving batched addurl and regular metadata calls\n    # causes multiple commits, so we do need to have it separate.\n    # TODO: figure out with Joey either could be avoided\n    from unittest.mock import patch\n    with patch.object(repo, \"always_commit\", False):\n        for filename, meta in add_metadata.items():\n            lgr.debug(\"Adding metadata to %s in %s\", filename, repo.path)\n            assert not repo.always_commit\n            for a in repo.set_metadata_(filename, add=meta):\n                res = annexjson2result(a, ds, type=\"file\", logger=lgr)\n                if res[\"status\"] == \"ok\":\n                    # Don't show all added metadata for the file because that\n                    # could quickly flood the output.\n                    del res[\"message\"]\n                yield res\n\n\n@build_doc\nclass Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'. [PY:  Alternatively, an iterable of dicts can be given.\n            PY]\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if isinstance(url_file, str):\n            if url_file != \"-\":\n                url_file = str(resolve_path(url_file, dataset))\n            try:\n                records, colidx_to_name = _read_from_file(\n                    url_file, input_type)\n            except ValueError as exc:\n                yield get_status_dict(action=\"addurls\",\n                                      ds=ds,\n                                      status=\"error\",\n                                      message=exc_str(exc))\n                return\n            displayed_source = \"'{}'\".format(urlfile)\n        else:\n            displayed_source = \"<records>\"\n            records = ensure_list(url_file)\n            colidx_to_name = {}\n\n        rows = None\n        if records:\n            try:\n                rows, subpaths = extract(records, colidx_to_name,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file={displayed_source}\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')\n\n\n__datalad_plugin__ = Addurls\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "1fe43d733",
                "new_commit_hash": "da56bf64f",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 399,
                            "length": 6
                        },
                        "new_range": {
                            "start": 399,
                            "length": 31
                        },
                        "section": "def _read(stream, input_type):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    return rows, idx_map",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def _read_from_file(fname, input_type):",
                                "type": "added"
                            },
                            {
                                "content": "    from_stdin = fname == \"-\"",
                                "type": "added"
                            },
                            {
                                "content": "    if input_type == \"ext\":",
                                "type": "added"
                            },
                            {
                                "content": "        if from_stdin:",
                                "type": "added"
                            },
                            {
                                "content": "            input_type = \"json\"",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            extension = os.path.splitext(fname)[1]",
                                "type": "added"
                            },
                            {
                                "content": "            if extension == \".json\":",
                                "type": "added"
                            },
                            {
                                "content": "                input_type = \"json\"",
                                "type": "added"
                            },
                            {
                                "content": "            elif extension == \".tsv\":",
                                "type": "added"
                            },
                            {
                                "content": "                input_type = \"tsv\"",
                                "type": "added"
                            },
                            {
                                "content": "            else:",
                                "type": "added"
                            },
                            {
                                "content": "                input_type = \"csv\"",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    fd = sys.stdin if from_stdin else open(fname)",
                                "type": "added"
                            },
                            {
                                "content": "    try:",
                                "type": "added"
                            },
                            {
                                "content": "        records, colidx_to_name = _read(fd, input_type)",
                                "type": "added"
                            },
                            {
                                "content": "        if not records:",
                                "type": "added"
                            },
                            {
                                "content": "            lgr.warning(\"No rows found in %s\", fd)",
                                "type": "added"
                            },
                            {
                                "content": "    finally:",
                                "type": "added"
                            },
                            {
                                "content": "        if fd is not sys.stdin:",
                                "type": "added"
                            },
                            {
                                "content": "            fd.close()",
                                "type": "added"
                            },
                            {
                                "content": "    return records, colidx_to_name",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "def _get_placeholder_exception(exc, msg_prefix, known):",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"Recast KeyError as a ValueError with close-match suggestions.",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "_read_from_file",
                            "content": "def _read_from_file(fname, input_type):\n    from_stdin = fname == \"-\"\n    if input_type == \"ext\":\n        if from_stdin:\n            input_type = \"json\"\n        else:\n            extension = os.path.splitext(fname)[1]\n            if extension == \".json\":\n                input_type = \"json\"\n            elif extension == \".tsv\":\n                input_type = \"tsv\"\n            else:\n                input_type = \"csv\"\n\n    fd = sys.stdin if from_stdin else open(fname)\n    try:\n        records, colidx_to_name = _read(fd, input_type)\n        if not records:\n            lgr.warning(\"No rows found in %s\", fd)\n    finally:\n        if fd is not sys.stdin:\n            fd.close()\n    return records, colidx_to_name",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 402,
                            "end_lineno": 424,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 540,
                            "length": 16
                        },
                        "new_range": {
                            "start": 565,
                            "length": 17
                        },
                        "section": "def sort_paths(paths):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    yield from sorted(paths, key=level_and_name)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",",
                                "type": "deleted"
                            },
                            {
                                "content": "def extract(rows, colidx_to_name=None,",
                                "type": "added"
                            },
                            {
                                "content": "            url_format=\"{0}\", filename_format=\"{1}\",",
                                "type": "added"
                            },
                            {
                                "content": "            exclude_autometa=None, meta=None, key=None,",
                                "type": "context"
                            },
                            {
                                "content": "            dry_run=False, missing_value=None):",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"Extract and format information from `url_file`.",
                                "type": "deleted"
                            },
                            {
                                "content": "    \"\"\"Extract and format information from `rows`.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    Parameters",
                                "type": "context"
                            },
                            {
                                "content": "    ----------",
                                "type": "context"
                            },
                            {
                                "content": "    stream : file object",
                                "type": "deleted"
                            },
                            {
                                "content": "        Items used to construct the file names and URLs.",
                                "type": "deleted"
                            },
                            {
                                "content": "    input_type : {'csv', 'tsv', 'json'}",
                                "type": "deleted"
                            },
                            {
                                "content": "    rows : list of dict",
                                "type": "added"
                            },
                            {
                                "content": "    colidx_to_name : dict, optional",
                                "type": "added"
                            },
                            {
                                "content": "        Mapping from a position index to a column name.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    All other parameters match those described in `AddUrls`.",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None, key=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'tsv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item a list subdataset paths,\n    sorted breadth-first.\n    \"\"\"\n    meta = ensure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n    if not rows:\n        lgr.warning(\"No rows found in %s\", stream)\n        return [], []\n\n    # Formatter for everything but file names\n    fmt = Formatter(colidx_to_name, missing_value)\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    info_fns = []\n    if formats_meta:\n        def set_meta_args(info, row):\n            info[\"meta_args\"] = clean_meta_args(fmt(row)\n                                                for fmt in formats_meta)\n        info_fns.append(set_meta_args)\n    if key:\n        key_parser = AnnexKeyParser(fmt.format, key)\n\n        def set_key(info, row):\n            info[\"key\"] = key_parser.parse(row)\n        info_fns.append(set_key)\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        try:\n            url = format_url(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in URL\", row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        info = {\"url\": url}\n        for fn in info_fns:\n            fn(info, row)\n        infos.append(info)\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, list(sort_paths(subpaths))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 543,
                            "end_lineno": 633,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 560,
                            "length": 11
                        },
                        "new_range": {
                            "start": 586,
                            "length": 7
                        },
                        "section": "def extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\","
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    sorted breadth-first.",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    meta = ensure_list(meta)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "    rows, colidx_to_name = _read(stream, input_type)",
                                "type": "deleted"
                            },
                            {
                                "content": "    if not rows:",
                                "type": "deleted"
                            },
                            {
                                "content": "        lgr.warning(\"No rows found in %s\", stream)",
                                "type": "deleted"
                            },
                            {
                                "content": "        return [], []",
                                "type": "deleted"
                            },
                            {
                                "content": "    colidx_to_name = colidx_to_name or {}",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # Formatter for everything but file names",
                                "type": "context"
                            },
                            {
                                "content": "    fmt = Formatter(colidx_to_name, missing_value)",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "function",
                            "name": "extract",
                            "content": "def extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None, key=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'tsv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item a list subdataset paths,\n    sorted breadth-first.\n    \"\"\"\n    meta = ensure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n    if not rows:\n        lgr.warning(\"No rows found in %s\", stream)\n        return [], []\n\n    # Formatter for everything but file names\n    fmt = Formatter(colidx_to_name, missing_value)\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    info_fns = []\n    if formats_meta:\n        def set_meta_args(info, row):\n            info[\"meta_args\"] = clean_meta_args(fmt(row)\n                                                for fmt in formats_meta)\n        info_fns.append(set_meta_args)\n    if key:\n        key_parser = AnnexKeyParser(fmt.format, key)\n\n        def set_key(info, row):\n            info[\"key\"] = key_parser.parse(row)\n        info_fns.append(set_key)\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        try:\n            url = format_url(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in URL\", row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        info = {\"url\": url}\n        for fn in info_fns:\n            fn(info, row)\n        infos.append(info)\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, list(sort_paths(subpaths))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 543,
                            "end_lineno": 633,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 978,
                            "length": 7
                        },
                        "new_range": {
                            "start": 1000,
                            "length": 8
                        },
                        "section": "class Addurls(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "            first row) or a JSON file (structured as a list of objects with",
                                "type": "context"
                            },
                            {
                                "content": "            string values). If '-', read from standard input, taking the",
                                "type": "context"
                            },
                            {
                                "content": "            content as JSON when --input-type is at its default value of",
                                "type": "context"
                            },
                            {
                                "content": "            'ext'.\"\"\"),",
                                "type": "deleted"
                            },
                            {
                                "content": "            'ext'. [PY:  Alternatively, an iterable of dicts can be given.",
                                "type": "added"
                            },
                            {
                                "content": "            PY]\"\"\"),",
                                "type": "added"
                            },
                            {
                                "content": "        urlformat=Parameter(",
                                "type": "context"
                            },
                            {
                                "content": "            args=(\"urlformat\",),",
                                "type": "context"
                            },
                            {
                                "content": "            metavar=\"URL-FORMAT\",",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "class",
                            "name": "Addurls",
                            "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 875,
                            "end_lineno": 1342,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1132,
                            "length": 25
                        },
                        "new_range": {
                            "start": 1155,
                            "length": 28
                        },
                        "section": "class Addurls(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                    yield dict(st_dict, status=\"error\", message=old_msg)",
                                "type": "context"
                            },
                            {
                                "content": "                    return",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if url_file != \"-\":",
                                "type": "deleted"
                            },
                            {
                                "content": "            url_file = str(resolve_path(url_file, dataset))",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "        if input_type == \"ext\":",
                                "type": "deleted"
                            },
                            {
                                "content": "            if url_file == \"-\":",
                                "type": "deleted"
                            },
                            {
                                "content": "                input_type = \"json\"",
                                "type": "deleted"
                            },
                            {
                                "content": "            else:",
                                "type": "deleted"
                            },
                            {
                                "content": "                extension = os.path.splitext(url_file)[1]",
                                "type": "deleted"
                            },
                            {
                                "content": "                if extension == \".json\":",
                                "type": "deleted"
                            },
                            {
                                "content": "                    input_type = \"json\"",
                                "type": "deleted"
                            },
                            {
                                "content": "                elif extension == \".tsv\":",
                                "type": "deleted"
                            },
                            {
                                "content": "                    input_type = \"tsv\"",
                                "type": "deleted"
                            },
                            {
                                "content": "                else:",
                                "type": "deleted"
                            },
                            {
                                "content": "                    input_type = \"csv\"",
                                "type": "deleted"
                            },
                            {
                                "content": "        if isinstance(url_file, str):",
                                "type": "added"
                            },
                            {
                                "content": "            if url_file != \"-\":",
                                "type": "added"
                            },
                            {
                                "content": "                url_file = str(resolve_path(url_file, dataset))",
                                "type": "added"
                            },
                            {
                                "content": "            try:",
                                "type": "added"
                            },
                            {
                                "content": "                records, colidx_to_name = _read_from_file(",
                                "type": "added"
                            },
                            {
                                "content": "                    url_file, input_type)",
                                "type": "added"
                            },
                            {
                                "content": "            except ValueError as exc:",
                                "type": "added"
                            },
                            {
                                "content": "                yield get_status_dict(action=\"addurls\",",
                                "type": "added"
                            },
                            {
                                "content": "                                      ds=ds,",
                                "type": "added"
                            },
                            {
                                "content": "                                      status=\"error\",",
                                "type": "added"
                            },
                            {
                                "content": "                                      message=exc_str(exc))",
                                "type": "added"
                            },
                            {
                                "content": "                return",
                                "type": "added"
                            },
                            {
                                "content": "            displayed_source = \"'{}'\".format(urlfile)",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            displayed_source = \"<records>\"",
                                "type": "added"
                            },
                            {
                                "content": "            records = ensure_list(url_file)",
                                "type": "added"
                            },
                            {
                                "content": "            colidx_to_name = {}",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        fd = sys.stdin if url_file == \"-\" else open(url_file)",
                                "type": "deleted"
                            },
                            {
                                "content": "        try:",
                                "type": "deleted"
                            },
                            {
                                "content": "        rows = None",
                                "type": "added"
                            },
                            {
                                "content": "        if records:",
                                "type": "added"
                            },
                            {
                                "content": "            try:",
                                "type": "context"
                            },
                            {
                                "content": "                rows, subpaths = extract(fd, input_type,",
                                "type": "deleted"
                            },
                            {
                                "content": "                rows, subpaths = extract(records, colidx_to_name,",
                                "type": "added"
                            },
                            {
                                "content": "                                         url_format, filename_format,",
                                "type": "context"
                            },
                            {
                                "content": "                                         exclude_autometa, meta, key,",
                                "type": "context"
                            },
                            {
                                "content": "                                         dry_run,",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "method",
                            "name": "Addurls.__call__",
                            "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 1098,
                            "end_lineno": 1342,
                            "parent": {
                                "file_name": "datalad/plugin/addurls.py",
                                "type": "class",
                                "name": "Addurls",
                                "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 875,
                                "end_lineno": 1342,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "class",
                            "name": "Addurls",
                            "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 875,
                            "end_lineno": 1342,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1158,
                            "length": 9
                        },
                        "new_range": {
                            "start": 1184,
                            "length": 6
                        },
                        "section": "class Addurls(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "            except (ValueError, RequestException) as exc:",
                                "type": "context"
                            },
                            {
                                "content": "                yield dict(st_dict, status=\"error\", message=exc_str(exc))",
                                "type": "context"
                            },
                            {
                                "content": "                return",
                                "type": "context"
                            },
                            {
                                "content": "        finally:",
                                "type": "deleted"
                            },
                            {
                                "content": "            if fd is not sys.stdin:",
                                "type": "deleted"
                            },
                            {
                                "content": "                fd.close()",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if not rows:",
                                "type": "context"
                            },
                            {
                                "content": "            yield dict(st_dict, status=\"notneeded\",",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "method",
                            "name": "Addurls.__call__",
                            "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 1098,
                            "end_lineno": 1342,
                            "parent": {
                                "file_name": "datalad/plugin/addurls.py",
                                "type": "class",
                                "name": "Addurls",
                                "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 875,
                                "end_lineno": 1342,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "class",
                            "name": "Addurls",
                            "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 875,
                            "end_lineno": 1342,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1329,
                            "length": 7
                        },
                        "new_range": {
                            "start": 1352,
                            "length": 7
                        },
                        "section": "class Addurls(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "{os.linesep.join(extra_msgs)}",
                                "type": "context"
                            },
                            {
                                "content": "url_file='{urlfile}'",
                                "type": "deleted"
                            },
                            {
                                "content": "url_file={displayed_source}",
                                "type": "added"
                            },
                            {
                                "content": "url_format='{url_format}'",
                                "type": "context"
                            },
                            {
                                "content": "filename_format='{filenameformat}'\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "method",
                            "name": "Addurls.__call__",
                            "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 1098,
                            "end_lineno": 1342,
                            "parent": {
                                "file_name": "datalad/plugin/addurls.py",
                                "type": "class",
                                "name": "Addurls",
                                "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 875,
                                "end_lineno": 1342,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/plugin/addurls.py",
                            "type": "class",
                            "name": "Addurls",
                            "content": "class Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 875,
                            "end_lineno": 1342,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport os.path as op\nimport shutil\nimport tempfile\n\nfrom unittest.mock import patch\n\nfrom io import StringIO\n\nfrom datalad.api import addurls, Dataset, subdatasets\nfrom datalad.cmd import WitlessRunner\nfrom datalad.consts import WEB_SPECIAL_REMOTE_UUID\nimport datalad.plugin.addurls as au\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.support.external_versions import external_versions\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_false,\n    assert_in,\n    assert_in_results,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_result_count,\n    assert_true,\n    chpwd,\n    create_tree,\n    eq_,\n    HTTPPath,\n    known_failure_githubci_win,\n    ok_exists,\n    ok_startswith,\n    skip_if,\n    SkipTest,\n    swallow_logs,\n    with_tempfile,\n    with_tree,\n    on_windows,\n    DEFAULT_BRANCH,\n)\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\nfrom datalad import cfg as dl_cfg\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = au.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = au.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = au.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = au.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = au.Formatter({})\n    eq_(fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}), \"value0\")\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = au.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    eq_(fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\"), \"'x'\")\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    eq_(fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\"), \"x    \")\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = au.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = au.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], {}),\n                         ([\" field=yes \"], {\"field\": \"yes\"}),\n                         ([\"field= value=\"], {\"field\": \"value=\"})]:\n        eq_(au.clean_meta_args(args), expect)\n\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"noequal\"])\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"=value\"])\n\n\ndef test_get_subpaths():\n    for fname, expect in [\n            (op.join(\"no\", \"dbl\", \"slash\"),\n             (op.join(\"no\", \"dbl\", \"slash\"), [])),\n            (\"p1//n\",\n             (op.join(\"p1\", \"n\"), [\"p1\"])),\n            (op.join(\"p1//p2\", \"p3//n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"n\"),\n              [\"p1\", op.join(\"p1\", \"p2\", \"p3\")])),\n            (op.join(\"p1//p2\", \"p3//p4\", \"p5//\", \"n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\", \"n\"),\n              [\"p1\",\n               op.join(\"p1\", \"p2\", \"p3\"),\n               op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\")])),\n            (\"//n\", (op.sep + \"n\", [\"\"])),\n            (\"n//\", (\"n\" + op.sep, [\"n\"]))]:\n        eq_(au.get_subpaths(fname), expect)\n\n\ndef test_sort_paths():\n    paths = [op.join(\"x\", \"a\", \"b\"),\n             \"z\",\n             op.join(\"y\", \"b\"),\n             op.join(\"y\", \"a\")]\n    expected = [\"z\",\n                op.join(\"y\", \"a\"),\n                op.join(\"y\", \"b\"),\n                op.join(\"x\", \"a\", \"b\")]\n    eq_(list(au.sort_paths(paths)), expected)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(au.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(au.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(au.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(au.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(au.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(au.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(au.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(au.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(au.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(au.fmt_to_name(\"\", {}))\n    assert_false(au.fmt_to_name(\"nonames\", {}))\n    assert_false(au.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_file_parts():\n    assert_dict_equal(au.get_file_parts(\"file.tar.gz\", \"prefix\"),\n                      {\"prefix\": \"file.tar.gz\",\n                       \"prefix_root_py\": \"file.tar\",\n                       \"prefix_ext_py\": \".gz\",\n                       \"prefix_root\": \"file\",\n                       \"prefix_ext\": \".tar.gz\"})\n\n\ndef test_get_url_parts():\n    eq_(au.get_url_parts(\"\"), {})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org\"),\n                      {\"_url_hostname\": \"datalad.org\"})\n\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      {\"_url_hostname\": \"datalad.org\",\n                       \"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\",\n                       \"_url_basename_root_py\": \"about\",\n                       \"_url_basename_ext_py\": \".html\",\n                       \"_url_basename_root\": \"about\",\n                       \"_url_basename_ext\": \".html\"})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      au.get_url_parts(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        au.get_url_parts(\"http://datalad.org/for/git-users\"),\n        {\"_url_hostname\": \"datalad.org\",\n         \"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\",\n         \"_url_basename_root_py\": \"git-users\",\n         \"_url_basename_ext_py\": \"\",\n         \"_url_basename_root\": \"git-users\",\n         \"_url_basename_ext\": \"\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\n@known_failure_githubci_win\ndef test_extract():\n    info, subpaths = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        [\"adult\", \"kid\", op.join(\"adult\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"kid\", \"no\")])\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [op.join(\"kid\", \"no\", \"will.csv\"), op.join(\"adult\", \"yes\", \"bob.csv\"),\n         op.join(\"adult\", \"no\", \"scott.csv\"), op.join(\"kid\", \"no\", \"max.csv\")])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [op.join(\"kid\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"adult\", \"no\"), op.join(\"kid\", \"no\")])\n\n\ndef test_extract_disable_autometa():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"*\",\n        meta=[\"group={age_group}\"])\n\n    eq_([d[\"meta_args\"] for d in info],\n        [{\"group\": \"kid\"}, {\"group\": \"adult\"}, {\"group\": \"adult\"},\n         {\"group\": \"kid\"}])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"ea\")\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\"},\n               {\"name\": \"max\", \"age_group\": \"kid\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n\ndef check_extract_csv_json_equal(input_type):\n    delim = \"\\t\" if input_type == \"tsv\" else \",\"\n\n    keys = ST_DATA[\"header\"]\n    csv_rows = [delim.join(keys)]\n    csv_rows.extend(delim.join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    kwds = dict(filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n                url_format=\"{name}_{debut_season}.com\",\n                meta=[\"group={age_group}\"])\n\n    json_output = au.extract(json_stream(ST_DATA[\"rows\"]), \"json\", **kwds)\n    csv_output = au.extract(csv_rows, input_type, **kwds)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_csv_tsv_json_equal():\n    yield check_extract_csv_json_equal, \"csv\"\n    yield check_extract_csv_json_equal, \"tsv\"\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  au.extract, None, \"invalid_input_type\")\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, annex=False)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.addurls(\"dummy_arg0\", \"dummy_arg1\", \"dummy_arg2\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tree({\"in.csv\": \"linky,abcd\\nhttps://datalad.org,f\"})\ndef test_addurls_unknown_placeholder(path):\n    ds = Dataset(path).create(force=True)\n    # Close but wrong URL placeholder\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{link}\", \"{abcd}\", dry_run=True)\n    assert_in(\"linky\", str(exc.exception))\n    # Close but wrong file name placeholder\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{abc}\", dry_run=True)\n    assert_in(\"abcd\", str(exc.exception))\n    # Out-of-bounds index.\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{3}\", dry_run=True)\n    assert_in(\"index\", str(exc.exception))\n\n    # Suggestions also work for automatic file name placeholders\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{_url_hostnam}\", dry_run=True)\n    assert_in(\"_url_hostname\", str(exc.exception))\n    # ... though if you whiff on the beginning prefix, we don't suggest\n    # anything because we decide to generate those fields based on detecting\n    # the prefix.\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{_uurl_hostnam}\", dry_run=True)\n    assert_not_in(\"_url_hostname\", str(exc.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    json_file = \"links.json\"\n    with open(op.join(ds.path, json_file), \"w\") as jfh:\n        json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                   {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                   {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                  jfh)\n\n    ds.save(message=\"setup\")\n\n    with swallow_logs(new_level=logging.INFO) as cml:\n        ds.addurls(json_file,\n                   \"{url}\",\n                   \"{subdir}//{_url_filename_root}\",\n                   dry_run=True)\n\n        for dir_ in [\"foo\", \"bar\"]:\n            assert_in(\"Would create a subdataset at {}\".format(dir_),\n                      cml.out)\n        assert_in(\n            \"Would download URL/a.dat to {}\".format(\n                os.path.join(path, \"foo\", \"BASE\")),\n            cml.out)\n\n        assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                  cml.out)\n\n\nOLD_EXAMINEKEY = external_versions[\"cmd:annex\"] < \"8.20201116\"\nskip_key_tests = skip_if(\n    OLD_EXAMINEKEY,\n    \"git-annex version does not support `examinekey --migrate-to-backend`\")\n\n\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.data = [{\"url\": cls.url + \"udir/a.dat\",\n                     \"name\": \"a\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"3fb7c40c70b0ed19da713bd69ee12014\",\n                     \"size\": \"9\"},\n                    {\"url\": cls.url + \"udir/b.dat\",\n                     \"name\": \"b\",\n                     \"subdir\": \"bar\",\n                     \"md5sum\": \"\",\n                     \"size\": \"\"},\n                    {\"url\": cls.url + \"udir/c.dat\",\n                     \"name\": \"c\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"9b72648021b70b8c522642e4490d7ac3\",\n                     \"size\": \"9\"}]\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(cls.data, jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return len(ds.repo.get_revisions(\"git-annex\"))\n\n        n_annex_commits = get_annex_commit_counts()\n\n        # Meanwhile also test that we can specify path relative\n        # to the top of the dataset, as we generally treat paths in\n        # Python API, and it will be the one saved in commit\n        # message record\n        json_file = op.relpath(self.json_file, ds.path)\n\n        ds.addurls(json_file, \"{url}\", \"{name}\",\n                   exclude_autometa=\"(md5sum|size)\")\n        ok_startswith(ds.repo.format_commit('%b', DEFAULT_BRANCH), f\"url_file='{json_file}'\")\n\n        filenames = [\"a\", \"b\", \"c\"]\n        for fname in filenames:\n            ok_exists(op.join(ds.path, fname))\n\n        for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                         [\"foo\", \"bar\", \"foo\"]):\n            assert_dict_equal(meta,\n                              {\"subdir\": [subdir], \"name\": [fname]})\n\n        # Ignore this check if we're faking dates because that disables\n        # batch mode.\n        # Also ignore if on Windows as it seems as if a git-annex bug \n        # leads to separate meta data commits: \n        # https://github.com/datalad/datalad/pull/5202#discussion_r535429704\n        if not (dl_cfg.get('datalad.fake-dates') or on_windows):\n            # We should have two new commits on the git-annex: one for the\n            # added urls and one for the added metadata.\n            eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n        # Add to already existing links, overwriting.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                       ifexists=\"overwrite\")\n            for fname in filenames:\n                assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                          cml.out)\n\n        # Add to already existing links, skipping.\n        assert_in_results(\n            ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n            action=\"addurls\",\n            status=\"notneeded\")\n\n        # Add to already existing links works, as long content is the same.\n        ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n        # But it fails if something has changed.\n        ds.unlock(\"a\")\n        with open(op.join(ds.path, \"a\"), \"w\") as ofh:\n            ofh.write(\"changed\")\n        ds.save(\"a\")\n\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_unbound_dataset(self, path):\n        def check(ds, dataset_arg, url_file, fname_format):\n            subdir = op.join(ds.path, \"subdir\")\n            os.mkdir(subdir)\n            with chpwd(subdir):\n                shutil.copy(self.json_file, \"in.json\")\n                addurls(dataset_arg, url_file, \"{url}\", fname_format)\n                # Files specified in the CSV file are always relative to the\n                # dataset.\n                for fname in [\"a\", \"b\", \"c\"]:\n                    ok_exists(op.join(ds.path, fname))\n\n        # The input file is relative to the current working directory, as\n        # with other commands.\n        ds0 = Dataset(op.join(path, \"ds0\")).create()\n        check(ds0, None, \"in.json\", \"{name}\")\n        # Likewise the input file is relative to the current working directory\n        # if a string dataset argument is given.\n        ds1 = Dataset(op.join(path, \"ds1\")).create()\n        check(ds1, ds1.path, \"in.json\", \"{name}\")\n        # A leading \"./\" doesn't confuse addurls() into downloading the file\n        # into the subdirectory.\n        ds2 = Dataset(op.join(path, \"ds2\")).create()\n        check(ds2, None, \"in.json\", \"./{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\",\n                cfg_proc=[\"yoda\"])\n        for fname in [\"a\", \"b\", \"c\", \"code\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        for save in True, False:\n            label = \"save\" if save else \"nosave\"\n            ds.addurls(self.json_file, \"{url}\",\n                       \"{subdir}-\" + label + \"//{name}\",\n                       save=save,\n                       cfg_proc=[\"yoda\"])\n\n            subdirs = [op.join(ds.path, \"{}-{}\".format(d, label))\n                       for d in [\"foo\", \"bar\"]]\n            subdir_files = dict(zip(subdirs, [[\"a\", \"c\"], [\"b\"]]))\n\n            for subds, fnames in subdir_files.items():\n                for fname in fnames:\n                    ok_exists(op.join(subds, fname))\n                # cfg_proc was applied generated subdatasets.\n                ok_exists(op.join(subds, \"code\"))\n            if save:\n                assert_repo_status(path)\n            else:\n                # The datasets are create but not saved (since asked not to)\n                assert_repo_status(path, untracked=subdirs)\n                # but the downloaded files aren't.\n                for subds, fnames in subdir_files.items():\n                    assert_repo_status(subds, added=fnames)\n\n        # Now save the \"--nosave\" changes and check that we have\n        # all the subdatasets.\n        ds.save()\n        eq_(set(subdatasets(dataset=ds, recursive=True,\n                            result_xfm=\"relpaths\")),\n            {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n        # We don't try to recreate existing subdatasets.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n            assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with assert_raises(IncompleteResultsError) as raised:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n        assert_in(\"There are file name collisions\", str(raised.exception))\n\n        ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n        for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file,\n                      \"{url}/nofilename/\",\n                      \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata_\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with patch.object(ds.repo, 'set_metadata_', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\",\n            jobs=3)\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n\n    @with_tree({\"in\": \"\"})\n    def test_addurls_invalid_input(self, path):\n        ds = Dataset(path).create(force=True)\n        in_file = op.join(path, \"in\")\n        for in_type in au.INPUT_TYPES:\n            with assert_raises(IncompleteResultsError) as exc:\n                ds.addurls(in_file, \"{url}\", \"{name}\", input_type=in_type)\n            assert_in(\"Failed to read\", str(exc.exception))\n\n    @with_tree({\"in.csv\": \"url,name,subdir\",\n                \"in.tsv\": \"url\\tname\\tsubdir\",\n                \"in.json\": \"[]\"})\n    def test_addurls_no_rows(self, path):\n        ds = Dataset(path).create(force=True)\n        for fname in [\"in.csv\", \"in.tsv\", \"in.json\"]:\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                assert_in_results(\n                    ds.addurls(fname, \"{url}\", \"{name}\"),\n                    action=\"addurls\",\n                    status=\"notneeded\")\n                cml.assert_logged(\"No rows\", regex=False)\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_stdin_input(self, input_text, input_type, path):\n        ds = Dataset(path).create(force=True)\n        with patch(\"sys.stdin\", new=StringIO(input_text)):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", input_type=input_type)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    def test_addurls_stdin_input(self):\n        def make_test(text, input_type, description):\n            def fn():\n                self.check_addurls_stdin_input(json_text, \"ext\")\n            fn.description = description\n            return fn\n\n        with open(self.json_file) as jfh:\n            json_text = jfh.read()\n\n        yield make_test(json_text, \"ext\", \"json,default input type\")\n        yield make_test(json_text, \"json\", \"json,json input type\")\n\n        def make_delim_text(delim):\n            row = \"{name}\" + delim + \"{url}\"\n            return \"\\n\".join(\n                [row.format(name=\"name\", url=\"url\")] +\n                [row.format(**rec) for rec in json.loads(json_text)])\n\n        yield make_test(make_delim_text(\",\"), \"csv\", \"csv,csv input type\")\n        yield make_test(make_delim_text(\"\\t\"), \"tsv\", \"tsv,tsv input type\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_stdin_input_command_line(self, path):\n        # The previous test checks all the cases, but it overrides sys.stdin.\n        # Do a simple check that's closer to a command line call.\n        Dataset(path).create(force=True)\n        runner = WitlessRunner(cwd=path)\n        with open(self.json_file) as jfh:\n            runner.run([\"datalad\", \"addurls\", '-', '{url}', '{name}'],\n                       stdin=jfh)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_drop_after(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.repo.set_gitattributes([('a*', {'annex.largefiles': 'nothing'})])\n        # make some files go to git, so we could test that we do not blow\n        # while trying to drop what is in git not annex\n        res = ds.addurls(self.json_file, '{url}', '{name}', drop_after=True)\n\n        assert_result_count(res, 3, action='addurl', status='ok')  # a, b, c  even if a goes to git\n        assert_result_count(res, 2, action='drop', status='ok')  # b, c\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_from_key_invalid_format(self, path):\n        ds = Dataset(path).create(force=True)\n        for fmt in [\"{name}-which-has-no-double-dash\",\n                    # Invalid hash length.\n                    \"MD5-s{size}--{md5sum}a\",\n                    # Invalid hash content.\n                    \"MD5-s{size}--\" + 32 * \"q\"]:\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           key=fmt, exclude_autometa=\"*\")\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_from_key(self, key_arg, expected_backend, fake_dates,\n                               path):\n        ds = Dataset(path).create(force=True, fake_dates=fake_dates)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        ds.addurls(self.json_file, \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                   key=key_arg)\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], expected_backend)\n            assert_false(pstat[\"has_content\"])\n\n        get_res = ds.get(paths, result_renderer=None, on_failure=\"ignore\")\n        assert_result_count(get_res, 2, action=\"get\", status=\"ok\")\n\n    def test_addurls_from_key(self):\n        fn = self.check_addurls_from_key\n        for case in [\n                (fn, \"MD5-s{size}--{md5sum}\", \"MD5\"),\n                (fn, \"MD5E-s{size}--{md5sum}.dat\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5-s{size}--{md5sum}\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5E-s{size}--{md5sum}.dat\", \"MD5\")]:\n            yield case + (False,)\n            yield case + (True,)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_row_missing_key_fields(self, path):\n        ds = Dataset(path).create(force=True)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        data = self.data.copy()\n        for row in data:\n            if row[\"name\"] == \"b\":\n                del row[\"md5sum\"]\n                break\n        with patch(\"sys.stdin\", new=StringIO(json.dumps(data))):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                       key=\"MD5-s{size}--{md5sum}\")\n\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], \"MD5\")\n            assert_false(pstat[\"has_content\"])\n",
            "new_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport os.path as op\nimport shutil\nimport tempfile\n\nfrom unittest.mock import patch\n\nfrom io import StringIO\n\nfrom datalad.api import addurls, Dataset, subdatasets\nfrom datalad.cmd import WitlessRunner\nfrom datalad.consts import WEB_SPECIAL_REMOTE_UUID\nimport datalad.plugin.addurls as au\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.support.external_versions import external_versions\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_false,\n    assert_in,\n    assert_in_results,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_result_count,\n    assert_true,\n    chpwd,\n    create_tree,\n    eq_,\n    HTTPPath,\n    known_failure_githubci_win,\n    ok_exists,\n    ok_startswith,\n    skip_if,\n    SkipTest,\n    swallow_logs,\n    with_tempfile,\n    with_tree,\n    on_windows,\n    DEFAULT_BRANCH,\n)\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\nfrom datalad import cfg as dl_cfg\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = au.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = au.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = au.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = au.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = au.Formatter({})\n    eq_(fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}), \"value0\")\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = au.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    eq_(fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\"), \"'x'\")\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    eq_(fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\"), \"x    \")\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = au.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = au.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], {}),\n                         ([\" field=yes \"], {\"field\": \"yes\"}),\n                         ([\"field= value=\"], {\"field\": \"value=\"})]:\n        eq_(au.clean_meta_args(args), expect)\n\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"noequal\"])\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"=value\"])\n\n\ndef test_get_subpaths():\n    for fname, expect in [\n            (op.join(\"no\", \"dbl\", \"slash\"),\n             (op.join(\"no\", \"dbl\", \"slash\"), [])),\n            (\"p1//n\",\n             (op.join(\"p1\", \"n\"), [\"p1\"])),\n            (op.join(\"p1//p2\", \"p3//n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"n\"),\n              [\"p1\", op.join(\"p1\", \"p2\", \"p3\")])),\n            (op.join(\"p1//p2\", \"p3//p4\", \"p5//\", \"n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\", \"n\"),\n              [\"p1\",\n               op.join(\"p1\", \"p2\", \"p3\"),\n               op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\")])),\n            (\"//n\", (op.sep + \"n\", [\"\"])),\n            (\"n//\", (\"n\" + op.sep, [\"n\"]))]:\n        eq_(au.get_subpaths(fname), expect)\n\n\ndef test_sort_paths():\n    paths = [op.join(\"x\", \"a\", \"b\"),\n             \"z\",\n             op.join(\"y\", \"b\"),\n             op.join(\"y\", \"a\")]\n    expected = [\"z\",\n                op.join(\"y\", \"a\"),\n                op.join(\"y\", \"b\"),\n                op.join(\"x\", \"a\", \"b\")]\n    eq_(list(au.sort_paths(paths)), expected)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(au.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(au.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(au.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(au.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(au.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(au.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(au.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(au.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(au.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(au.fmt_to_name(\"\", {}))\n    assert_false(au.fmt_to_name(\"nonames\", {}))\n    assert_false(au.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_file_parts():\n    assert_dict_equal(au.get_file_parts(\"file.tar.gz\", \"prefix\"),\n                      {\"prefix\": \"file.tar.gz\",\n                       \"prefix_root_py\": \"file.tar\",\n                       \"prefix_ext_py\": \".gz\",\n                       \"prefix_root\": \"file\",\n                       \"prefix_ext\": \".tar.gz\"})\n\n\ndef test_get_url_parts():\n    eq_(au.get_url_parts(\"\"), {})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org\"),\n                      {\"_url_hostname\": \"datalad.org\"})\n\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      {\"_url_hostname\": \"datalad.org\",\n                       \"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\",\n                       \"_url_basename_root_py\": \"about\",\n                       \"_url_basename_ext_py\": \".html\",\n                       \"_url_basename_root\": \"about\",\n                       \"_url_basename_ext\": \".html\"})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      au.get_url_parts(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        au.get_url_parts(\"http://datalad.org/for/git-users\"),\n        {\"_url_hostname\": \"datalad.org\",\n         \"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\",\n         \"_url_basename_root_py\": \"git-users\",\n         \"_url_basename_ext_py\": \"\",\n         \"_url_basename_root\": \"git-users\",\n         \"_url_basename_ext\": \"\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\n@known_failure_githubci_win\ndef test_extract():\n    info, subpaths = au.extract(\n        ST_DATA[\"rows\"],\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        [\"adult\", \"kid\", op.join(\"adult\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"kid\", \"no\")])\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [op.join(\"kid\", \"no\", \"will.csv\"), op.join(\"adult\", \"yes\", \"bob.csv\"),\n         op.join(\"adult\", \"no\", \"scott.csv\"), op.join(\"kid\", \"no\", \"max.csv\")])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [op.join(\"kid\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"adult\", \"no\"), op.join(\"kid\", \"no\")])\n\n\ndef test_extract_disable_autometa():\n    info, _ = au.extract(\n        ST_DATA[\"rows\"],\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"*\",\n        meta=[\"group={age_group}\"])\n\n    eq_([d[\"meta_args\"] for d in info],\n        [{\"group\": \"kid\"}, {\"group\": \"adult\"}, {\"group\": \"adult\"},\n         {\"group\": \"kid\"}])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, _ = au.extract(\n        ST_DATA[\"rows\"],\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"ea\")\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\"},\n               {\"name\": \"max\", \"age_group\": \"kid\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n\ndef check_extract_csv_json_equal(input_type):\n    delim = \"\\t\" if input_type == \"tsv\" else \",\"\n\n    keys = ST_DATA[\"header\"]\n    csv_rows = [delim.join(keys)]\n    csv_rows.extend(delim.join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    kwds = dict(filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n                url_format=\"{name}_{debut_season}.com\",\n                meta=[\"group={age_group}\"])\n\n    json_output = au.extract(\n        *au._read(json_stream(ST_DATA[\"rows\"]), \"json\"), **kwds)\n    csv_output = au.extract(\n        *au._read(csv_rows, input_type), **kwds)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_csv_tsv_json_equal():\n    yield check_extract_csv_json_equal, \"csv\"\n    yield check_extract_csv_json_equal, \"tsv\"\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  au._read, None, \"invalid_input_type\")\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, annex=False)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.addurls(\"dummy_arg0\", \"dummy_arg1\", \"dummy_arg2\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tree({\"in.csv\": \"linky,abcd\\nhttps://datalad.org,f\"})\ndef test_addurls_unknown_placeholder(path):\n    ds = Dataset(path).create(force=True)\n    # Close but wrong URL placeholder\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{link}\", \"{abcd}\", dry_run=True)\n    assert_in(\"linky\", str(exc.exception))\n    # Close but wrong file name placeholder\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{abc}\", dry_run=True)\n    assert_in(\"abcd\", str(exc.exception))\n    # Out-of-bounds index.\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{3}\", dry_run=True)\n    assert_in(\"index\", str(exc.exception))\n\n    # Suggestions also work for automatic file name placeholders\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{_url_hostnam}\", dry_run=True)\n    assert_in(\"_url_hostname\", str(exc.exception))\n    # ... though if you whiff on the beginning prefix, we don't suggest\n    # anything because we decide to generate those fields based on detecting\n    # the prefix.\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{_uurl_hostnam}\", dry_run=True)\n    assert_not_in(\"_url_hostname\", str(exc.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    json_file = \"links.json\"\n    with open(op.join(ds.path, json_file), \"w\") as jfh:\n        json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                   {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                   {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                  jfh)\n\n    ds.save(message=\"setup\")\n\n    with swallow_logs(new_level=logging.INFO) as cml:\n        ds.addurls(json_file,\n                   \"{url}\",\n                   \"{subdir}//{_url_filename_root}\",\n                   dry_run=True)\n\n        for dir_ in [\"foo\", \"bar\"]:\n            assert_in(\"Would create a subdataset at {}\".format(dir_),\n                      cml.out)\n        assert_in(\n            \"Would download URL/a.dat to {}\".format(\n                os.path.join(path, \"foo\", \"BASE\")),\n            cml.out)\n\n        assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                  cml.out)\n\n\nOLD_EXAMINEKEY = external_versions[\"cmd:annex\"] < \"8.20201116\"\nskip_key_tests = skip_if(\n    OLD_EXAMINEKEY,\n    \"git-annex version does not support `examinekey --migrate-to-backend`\")\n\n\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.data = [{\"url\": cls.url + \"udir/a.dat\",\n                     \"name\": \"a\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"3fb7c40c70b0ed19da713bd69ee12014\",\n                     \"size\": \"9\"},\n                    {\"url\": cls.url + \"udir/b.dat\",\n                     \"name\": \"b\",\n                     \"subdir\": \"bar\",\n                     \"md5sum\": \"\",\n                     \"size\": \"\"},\n                    {\"url\": cls.url + \"udir/c.dat\",\n                     \"name\": \"c\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"9b72648021b70b8c522642e4490d7ac3\",\n                     \"size\": \"9\"}]\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(cls.data, jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return len(ds.repo.get_revisions(\"git-annex\"))\n\n        n_annex_commits = get_annex_commit_counts()\n\n        # Meanwhile also test that we can specify path relative\n        # to the top of the dataset, as we generally treat paths in\n        # Python API, and it will be the one saved in commit\n        # message record\n        json_file = op.relpath(self.json_file, ds.path)\n\n        ds.addurls(json_file, \"{url}\", \"{name}\",\n                   exclude_autometa=\"(md5sum|size)\")\n        ok_startswith(ds.repo.format_commit('%b', DEFAULT_BRANCH), f\"url_file='{json_file}'\")\n\n        filenames = [\"a\", \"b\", \"c\"]\n        for fname in filenames:\n            ok_exists(op.join(ds.path, fname))\n\n        for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                         [\"foo\", \"bar\", \"foo\"]):\n            assert_dict_equal(meta,\n                              {\"subdir\": [subdir], \"name\": [fname]})\n\n        # Ignore this check if we're faking dates because that disables\n        # batch mode.\n        # Also ignore if on Windows as it seems as if a git-annex bug \n        # leads to separate meta data commits: \n        # https://github.com/datalad/datalad/pull/5202#discussion_r535429704\n        if not (dl_cfg.get('datalad.fake-dates') or on_windows):\n            # We should have two new commits on the git-annex: one for the\n            # added urls and one for the added metadata.\n            eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n        # Add to already existing links, overwriting.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                       ifexists=\"overwrite\")\n            for fname in filenames:\n                assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                          cml.out)\n\n        # Add to already existing links, skipping.\n        assert_in_results(\n            ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n            action=\"addurls\",\n            status=\"notneeded\")\n\n        # Add to already existing links works, as long content is the same.\n        ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n        # But it fails if something has changed.\n        ds.unlock(\"a\")\n        with open(op.join(ds.path, \"a\"), \"w\") as ofh:\n            ofh.write(\"changed\")\n        ds.save(\"a\")\n\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_unbound_dataset(self, path):\n        def check(ds, dataset_arg, url_file, fname_format):\n            subdir = op.join(ds.path, \"subdir\")\n            os.mkdir(subdir)\n            with chpwd(subdir):\n                shutil.copy(self.json_file, \"in.json\")\n                addurls(dataset_arg, url_file, \"{url}\", fname_format)\n                # Files specified in the CSV file are always relative to the\n                # dataset.\n                for fname in [\"a\", \"b\", \"c\"]:\n                    ok_exists(op.join(ds.path, fname))\n\n        # The input file is relative to the current working directory, as\n        # with other commands.\n        ds0 = Dataset(op.join(path, \"ds0\")).create()\n        check(ds0, None, \"in.json\", \"{name}\")\n        # Likewise the input file is relative to the current working directory\n        # if a string dataset argument is given.\n        ds1 = Dataset(op.join(path, \"ds1\")).create()\n        check(ds1, ds1.path, \"in.json\", \"{name}\")\n        # A leading \"./\" doesn't confuse addurls() into downloading the file\n        # into the subdirectory.\n        ds2 = Dataset(op.join(path, \"ds2\")).create()\n        check(ds2, None, \"in.json\", \"./{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\",\n                cfg_proc=[\"yoda\"])\n        for fname in [\"a\", \"b\", \"c\", \"code\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile\n    def test_addurls_from_list(self, path):\n        ds = Dataset(path).create()\n        ds.addurls(self.data, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        for save in True, False:\n            label = \"save\" if save else \"nosave\"\n            ds.addurls(self.json_file, \"{url}\",\n                       \"{subdir}-\" + label + \"//{name}\",\n                       save=save,\n                       cfg_proc=[\"yoda\"])\n\n            subdirs = [op.join(ds.path, \"{}-{}\".format(d, label))\n                       for d in [\"foo\", \"bar\"]]\n            subdir_files = dict(zip(subdirs, [[\"a\", \"c\"], [\"b\"]]))\n\n            for subds, fnames in subdir_files.items():\n                for fname in fnames:\n                    ok_exists(op.join(subds, fname))\n                # cfg_proc was applied generated subdatasets.\n                ok_exists(op.join(subds, \"code\"))\n            if save:\n                assert_repo_status(path)\n            else:\n                # The datasets are create but not saved (since asked not to)\n                assert_repo_status(path, untracked=subdirs)\n                # but the downloaded files aren't.\n                for subds, fnames in subdir_files.items():\n                    assert_repo_status(subds, added=fnames)\n\n        # Now save the \"--nosave\" changes and check that we have\n        # all the subdatasets.\n        ds.save()\n        eq_(set(subdatasets(dataset=ds, recursive=True,\n                            result_xfm=\"relpaths\")),\n            {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n        # We don't try to recreate existing subdatasets.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n            assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with assert_raises(IncompleteResultsError) as raised:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n        assert_in(\"There are file name collisions\", str(raised.exception))\n\n        ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n        for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file,\n                      \"{url}/nofilename/\",\n                      \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata_\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with patch.object(ds.repo, 'set_metadata_', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\",\n            jobs=3)\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n\n    @with_tree({\"in\": \"\"})\n    def test_addurls_invalid_input(self, path):\n        ds = Dataset(path).create(force=True)\n        in_file = op.join(path, \"in\")\n        for in_type in au.INPUT_TYPES:\n            with assert_raises(IncompleteResultsError) as exc:\n                ds.addurls(in_file, \"{url}\", \"{name}\", input_type=in_type)\n            assert_in(\"Failed to read\", str(exc.exception))\n\n    @with_tree({\"in.csv\": \"url,name,subdir\",\n                \"in.tsv\": \"url\\tname\\tsubdir\",\n                \"in.json\": \"[]\"})\n    def test_addurls_no_rows(self, path):\n        ds = Dataset(path).create(force=True)\n        for fname in [\"in.csv\", \"in.tsv\", \"in.json\"]:\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                assert_in_results(\n                    ds.addurls(fname, \"{url}\", \"{name}\"),\n                    action=\"addurls\",\n                    status=\"notneeded\")\n                cml.assert_logged(\"No rows\", regex=False)\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_stdin_input(self, input_text, input_type, path):\n        ds = Dataset(path).create(force=True)\n        with patch(\"sys.stdin\", new=StringIO(input_text)):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", input_type=input_type)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    def test_addurls_stdin_input(self):\n        def make_test(text, input_type, description):\n            def fn():\n                self.check_addurls_stdin_input(json_text, \"ext\")\n            fn.description = description\n            return fn\n\n        with open(self.json_file) as jfh:\n            json_text = jfh.read()\n\n        yield make_test(json_text, \"ext\", \"json,default input type\")\n        yield make_test(json_text, \"json\", \"json,json input type\")\n\n        def make_delim_text(delim):\n            row = \"{name}\" + delim + \"{url}\"\n            return \"\\n\".join(\n                [row.format(name=\"name\", url=\"url\")] +\n                [row.format(**rec) for rec in json.loads(json_text)])\n\n        yield make_test(make_delim_text(\",\"), \"csv\", \"csv,csv input type\")\n        yield make_test(make_delim_text(\"\\t\"), \"tsv\", \"tsv,tsv input type\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_stdin_input_command_line(self, path):\n        # The previous test checks all the cases, but it overrides sys.stdin.\n        # Do a simple check that's closer to a command line call.\n        Dataset(path).create(force=True)\n        runner = WitlessRunner(cwd=path)\n        with open(self.json_file) as jfh:\n            runner.run([\"datalad\", \"addurls\", '-', '{url}', '{name}'],\n                       stdin=jfh)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_drop_after(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.repo.set_gitattributes([('a*', {'annex.largefiles': 'nothing'})])\n        # make some files go to git, so we could test that we do not blow\n        # while trying to drop what is in git not annex\n        res = ds.addurls(self.json_file, '{url}', '{name}', drop_after=True)\n\n        assert_result_count(res, 3, action='addurl', status='ok')  # a, b, c  even if a goes to git\n        assert_result_count(res, 2, action='drop', status='ok')  # b, c\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_from_key_invalid_format(self, path):\n        ds = Dataset(path).create(force=True)\n        for fmt in [\"{name}-which-has-no-double-dash\",\n                    # Invalid hash length.\n                    \"MD5-s{size}--{md5sum}a\",\n                    # Invalid hash content.\n                    \"MD5-s{size}--\" + 32 * \"q\"]:\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           key=fmt, exclude_autometa=\"*\")\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_from_key(self, key_arg, expected_backend, fake_dates,\n                               path):\n        ds = Dataset(path).create(force=True, fake_dates=fake_dates)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        ds.addurls(self.json_file, \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                   key=key_arg)\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], expected_backend)\n            assert_false(pstat[\"has_content\"])\n\n        get_res = ds.get(paths, result_renderer=None, on_failure=\"ignore\")\n        assert_result_count(get_res, 2, action=\"get\", status=\"ok\")\n\n    def test_addurls_from_key(self):\n        fn = self.check_addurls_from_key\n        for case in [\n                (fn, \"MD5-s{size}--{md5sum}\", \"MD5\"),\n                (fn, \"MD5E-s{size}--{md5sum}.dat\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5-s{size}--{md5sum}\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5E-s{size}--{md5sum}.dat\", \"MD5\")]:\n            yield case + (False,)\n            yield case + (True,)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_row_missing_key_fields(self, path):\n        ds = Dataset(path).create(force=True)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        data = self.data.copy()\n        for row in data:\n            if row[\"name\"] == \"b\":\n                del row[\"md5sum\"]\n                break\n        with patch(\"sys.stdin\", new=StringIO(json.dumps(data))):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                       key=\"MD5-s{size}--{md5sum}\")\n\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], \"MD5\")\n            assert_false(pstat[\"has_content\"])\n",
            "header": {
                "file": {
                    "path": "datalad/plugin/tests/test_addurls.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "90771b555",
                "new_commit_hash": "89fab23f7",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/plugin/tests/test_addurls.py"
            },
            "plus_file": {
                "path": "b/datalad/plugin/tests/test_addurls.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 256,
                            "length": 7
                        },
                        "new_range": {
                            "start": 256,
                            "length": 7
                        },
                        "section": "def json_stream(data):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "@known_failure_githubci_win",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract():",
                                "type": "context"
                            },
                            {
                                "content": "    info, subpaths = au.extract(",
                                "type": "context"
                            },
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "deleted"
                            },
                            {
                                "content": "        ST_DATA[\"rows\"],",
                                "type": "added"
                            },
                            {
                                "content": "        url_format=\"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "        filename_format=\"{age_group}//{now_dead}//{name}.csv\")",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract",
                            "content": "def test_extract():\n    info, subpaths = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        [\"adult\", \"kid\", op.join(\"adult\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"kid\", \"no\")])\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [op.join(\"kid\", \"no\", \"will.csv\"), op.join(\"adult\", \"yes\", \"bob.csv\"),\n         op.join(\"adult\", \"no\", \"scott.csv\"), op.join(\"kid\", \"no\", \"max.csv\")])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [op.join(\"kid\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"adult\", \"no\"), op.join(\"kid\", \"no\")])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 257,
                            "end_lineno": 285,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 287,
                            "length": 7
                        },
                        "new_range": {
                            "start": 287,
                            "length": 7
                        },
                        "section": "def test_extract():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract_disable_autometa():",
                                "type": "context"
                            },
                            {
                                "content": "    info, _ = au.extract(",
                                "type": "context"
                            },
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "deleted"
                            },
                            {
                                "content": "        ST_DATA[\"rows\"],",
                                "type": "added"
                            },
                            {
                                "content": "        url_format=\"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "        filename_format=\"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "        exclude_autometa=\"*\",",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_disable_autometa",
                            "content": "def test_extract_disable_autometa():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"*\",\n        meta=[\"group={age_group}\"])\n\n    eq_([d[\"meta_args\"] for d in info],\n        [{\"group\": \"kid\"}, {\"group\": \"adult\"}, {\"group\": \"adult\"},\n         {\"group\": \"kid\"}])",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 288,
                            "end_lineno": 298,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 300,
                            "length": 7
                        },
                        "new_range": {
                            "start": 300,
                            "length": 7
                        },
                        "section": "def test_extract_disable_autometa():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract_exclude_autometa_regexp():",
                                "type": "context"
                            },
                            {
                                "content": "    info, _ = au.extract(",
                                "type": "context"
                            },
                            {
                                "content": "        json_stream(ST_DATA[\"rows\"]), \"json\",",
                                "type": "deleted"
                            },
                            {
                                "content": "        ST_DATA[\"rows\"],",
                                "type": "added"
                            },
                            {
                                "content": "        url_format=\"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "        filename_format=\"{age_group}//{now_dead}//{name}.csv\",",
                                "type": "context"
                            },
                            {
                                "content": "        exclude_autometa=\"ea\")",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_exclude_autometa_regexp",
                            "content": "def test_extract_exclude_autometa_regexp():\n    info, _ = au.extract(\n        json_stream(ST_DATA[\"rows\"]), \"json\",\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"ea\")\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\"},\n               {\"name\": \"max\", \"age_group\": \"kid\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 301,
                            "end_lineno": 313,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 325,
                            "length": 8
                        },
                        "new_range": {
                            "start": 325,
                            "length": 10
                        },
                        "section": "def check_extract_csv_json_equal(input_type):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                url_format=\"{name}_{debut_season}.com\",",
                                "type": "context"
                            },
                            {
                                "content": "                meta=[\"group={age_group}\"])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    json_output = au.extract(json_stream(ST_DATA[\"rows\"]), \"json\", **kwds)",
                                "type": "deleted"
                            },
                            {
                                "content": "    csv_output = au.extract(csv_rows, input_type, **kwds)",
                                "type": "deleted"
                            },
                            {
                                "content": "    json_output = au.extract(",
                                "type": "added"
                            },
                            {
                                "content": "        *au._read(json_stream(ST_DATA[\"rows\"]), \"json\"), **kwds)",
                                "type": "added"
                            },
                            {
                                "content": "    csv_output = au.extract(",
                                "type": "added"
                            },
                            {
                                "content": "        *au._read(csv_rows, input_type), **kwds)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(json_output, csv_output)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "check_extract_csv_json_equal",
                            "content": "def check_extract_csv_json_equal(input_type):\n    delim = \"\\t\" if input_type == \"tsv\" else \",\"\n\n    keys = ST_DATA[\"header\"]\n    csv_rows = [delim.join(keys)]\n    csv_rows.extend(delim.join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    kwds = dict(filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n                url_format=\"{name}_{debut_season}.com\",\n                meta=[\"group={age_group}\"])\n\n    json_output = au.extract(json_stream(ST_DATA[\"rows\"]), \"json\", **kwds)\n    csv_output = au.extract(csv_rows, input_type, **kwds)\n\n    eq_(json_output, csv_output)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 316,
                            "end_lineno": 331,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 338,
                            "length": 7
                        },
                        "new_range": {
                            "start": 340,
                            "length": 7
                        },
                        "section": "def test_extract_csv_tsv_json_equal():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_extract_wrong_input_type():",
                                "type": "context"
                            },
                            {
                                "content": "    assert_raises(ValueError,",
                                "type": "context"
                            },
                            {
                                "content": "                  au.extract, None, \"invalid_input_type\")",
                                "type": "deleted"
                            },
                            {
                                "content": "                  au._read, None, \"invalid_input_type\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@with_tempfile(mkdir=True)",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "function",
                            "name": "test_extract_wrong_input_type",
                            "content": "def test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  au.extract, None, \"invalid_input_type\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 339,
                            "end_lineno": 341,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 552,
                            "length": 6
                        },
                        "new_range": {
                            "start": 554,
                            "length": 13
                        },
                        "section": "class TestAddurls(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        for fname in [\"a\", \"b\", \"c\", \"code\"]:",
                                "type": "context"
                            },
                            {
                                "content": "            ok_exists(os.path.join(dspath, fname))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    @with_tempfile",
                                "type": "added"
                            },
                            {
                                "content": "    def test_addurls_from_list(self, path):",
                                "type": "added"
                            },
                            {
                                "content": "        ds = Dataset(path).create()",
                                "type": "added"
                            },
                            {
                                "content": "        ds.addurls(self.data, \"{url}\", \"{name}\")",
                                "type": "added"
                            },
                            {
                                "content": "        for fname in [\"a\", \"b\", \"c\"]:",
                                "type": "added"
                            },
                            {
                                "content": "            ok_exists(op.join(path, fname))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    @with_tempfile(mkdir=True)",
                                "type": "context"
                            },
                            {
                                "content": "    def test_addurls_subdataset(self, path):",
                                "type": "context"
                            },
                            {
                                "content": "        ds = Dataset(path).create(force=True)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "class",
                            "name": "TestAddurls",
                            "content": "class TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.data = [{\"url\": cls.url + \"udir/a.dat\",\n                     \"name\": \"a\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"3fb7c40c70b0ed19da713bd69ee12014\",\n                     \"size\": \"9\"},\n                    {\"url\": cls.url + \"udir/b.dat\",\n                     \"name\": \"b\",\n                     \"subdir\": \"bar\",\n                     \"md5sum\": \"\",\n                     \"size\": \"\"},\n                    {\"url\": cls.url + \"udir/c.dat\",\n                     \"name\": \"c\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"9b72648021b70b8c522642e4490d7ac3\",\n                     \"size\": \"9\"}]\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(cls.data, jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return len(ds.repo.get_revisions(\"git-annex\"))\n\n        n_annex_commits = get_annex_commit_counts()\n\n        # Meanwhile also test that we can specify path relative\n        # to the top of the dataset, as we generally treat paths in\n        # Python API, and it will be the one saved in commit\n        # message record\n        json_file = op.relpath(self.json_file, ds.path)\n\n        ds.addurls(json_file, \"{url}\", \"{name}\",\n                   exclude_autometa=\"(md5sum|size)\")\n        ok_startswith(ds.repo.format_commit('%b', DEFAULT_BRANCH), f\"url_file='{json_file}'\")\n\n        filenames = [\"a\", \"b\", \"c\"]\n        for fname in filenames:\n            ok_exists(op.join(ds.path, fname))\n\n        for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                         [\"foo\", \"bar\", \"foo\"]):\n            assert_dict_equal(meta,\n                              {\"subdir\": [subdir], \"name\": [fname]})\n\n        # Ignore this check if we're faking dates because that disables\n        # batch mode.\n        # Also ignore if on Windows as it seems as if a git-annex bug \n        # leads to separate meta data commits: \n        # https://github.com/datalad/datalad/pull/5202#discussion_r535429704\n        if not (dl_cfg.get('datalad.fake-dates') or on_windows):\n            # We should have two new commits on the git-annex: one for the\n            # added urls and one for the added metadata.\n            eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n        # Add to already existing links, overwriting.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                       ifexists=\"overwrite\")\n            for fname in filenames:\n                assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                          cml.out)\n\n        # Add to already existing links, skipping.\n        assert_in_results(\n            ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n            action=\"addurls\",\n            status=\"notneeded\")\n\n        # Add to already existing links works, as long content is the same.\n        ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n        # But it fails if something has changed.\n        ds.unlock(\"a\")\n        with open(op.join(ds.path, \"a\"), \"w\") as ofh:\n            ofh.write(\"changed\")\n        ds.save(\"a\")\n\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_unbound_dataset(self, path):\n        def check(ds, dataset_arg, url_file, fname_format):\n            subdir = op.join(ds.path, \"subdir\")\n            os.mkdir(subdir)\n            with chpwd(subdir):\n                shutil.copy(self.json_file, \"in.json\")\n                addurls(dataset_arg, url_file, \"{url}\", fname_format)\n                # Files specified in the CSV file are always relative to the\n                # dataset.\n                for fname in [\"a\", \"b\", \"c\"]:\n                    ok_exists(op.join(ds.path, fname))\n\n        # The input file is relative to the current working directory, as\n        # with other commands.\n        ds0 = Dataset(op.join(path, \"ds0\")).create()\n        check(ds0, None, \"in.json\", \"{name}\")\n        # Likewise the input file is relative to the current working directory\n        # if a string dataset argument is given.\n        ds1 = Dataset(op.join(path, \"ds1\")).create()\n        check(ds1, ds1.path, \"in.json\", \"{name}\")\n        # A leading \"./\" doesn't confuse addurls() into downloading the file\n        # into the subdirectory.\n        ds2 = Dataset(op.join(path, \"ds2\")).create()\n        check(ds2, None, \"in.json\", \"./{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\",\n                cfg_proc=[\"yoda\"])\n        for fname in [\"a\", \"b\", \"c\", \"code\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile\n    def test_addurls_from_list(self, path):\n        ds = Dataset(path).create()\n        ds.addurls(self.data, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        for save in True, False:\n            label = \"save\" if save else \"nosave\"\n            ds.addurls(self.json_file, \"{url}\",\n                       \"{subdir}-\" + label + \"//{name}\",\n                       save=save,\n                       cfg_proc=[\"yoda\"])\n\n            subdirs = [op.join(ds.path, \"{}-{}\".format(d, label))\n                       for d in [\"foo\", \"bar\"]]\n            subdir_files = dict(zip(subdirs, [[\"a\", \"c\"], [\"b\"]]))\n\n            for subds, fnames in subdir_files.items():\n                for fname in fnames:\n                    ok_exists(op.join(subds, fname))\n                # cfg_proc was applied generated subdatasets.\n                ok_exists(op.join(subds, \"code\"))\n            if save:\n                assert_repo_status(path)\n            else:\n                # The datasets are create but not saved (since asked not to)\n                assert_repo_status(path, untracked=subdirs)\n                # but the downloaded files aren't.\n                for subds, fnames in subdir_files.items():\n                    assert_repo_status(subds, added=fnames)\n\n        # Now save the \"--nosave\" changes and check that we have\n        # all the subdatasets.\n        ds.save()\n        eq_(set(subdatasets(dataset=ds, recursive=True,\n                            result_xfm=\"relpaths\")),\n            {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n        # We don't try to recreate existing subdatasets.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n            assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with assert_raises(IncompleteResultsError) as raised:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n        assert_in(\"There are file name collisions\", str(raised.exception))\n\n        ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n        for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file,\n                      \"{url}/nofilename/\",\n                      \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata_\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with patch.object(ds.repo, 'set_metadata_', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\",\n            jobs=3)\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n\n    @with_tree({\"in\": \"\"})\n    def test_addurls_invalid_input(self, path):\n        ds = Dataset(path).create(force=True)\n        in_file = op.join(path, \"in\")\n        for in_type in au.INPUT_TYPES:\n            with assert_raises(IncompleteResultsError) as exc:\n                ds.addurls(in_file, \"{url}\", \"{name}\", input_type=in_type)\n            assert_in(\"Failed to read\", str(exc.exception))\n\n    @with_tree({\"in.csv\": \"url,name,subdir\",\n                \"in.tsv\": \"url\\tname\\tsubdir\",\n                \"in.json\": \"[]\"})\n    def test_addurls_no_rows(self, path):\n        ds = Dataset(path).create(force=True)\n        for fname in [\"in.csv\", \"in.tsv\", \"in.json\"]:\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                assert_in_results(\n                    ds.addurls(fname, \"{url}\", \"{name}\"),\n                    action=\"addurls\",\n                    status=\"notneeded\")\n                cml.assert_logged(\"No rows\", regex=False)\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_stdin_input(self, input_text, input_type, path):\n        ds = Dataset(path).create(force=True)\n        with patch(\"sys.stdin\", new=StringIO(input_text)):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", input_type=input_type)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    def test_addurls_stdin_input(self):\n        def make_test(text, input_type, description):\n            def fn():\n                self.check_addurls_stdin_input(json_text, \"ext\")\n            fn.description = description\n            return fn\n\n        with open(self.json_file) as jfh:\n            json_text = jfh.read()\n\n        yield make_test(json_text, \"ext\", \"json,default input type\")\n        yield make_test(json_text, \"json\", \"json,json input type\")\n\n        def make_delim_text(delim):\n            row = \"{name}\" + delim + \"{url}\"\n            return \"\\n\".join(\n                [row.format(name=\"name\", url=\"url\")] +\n                [row.format(**rec) for rec in json.loads(json_text)])\n\n        yield make_test(make_delim_text(\",\"), \"csv\", \"csv,csv input type\")\n        yield make_test(make_delim_text(\"\\t\"), \"tsv\", \"tsv,tsv input type\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_stdin_input_command_line(self, path):\n        # The previous test checks all the cases, but it overrides sys.stdin.\n        # Do a simple check that's closer to a command line call.\n        Dataset(path).create(force=True)\n        runner = WitlessRunner(cwd=path)\n        with open(self.json_file) as jfh:\n            runner.run([\"datalad\", \"addurls\", '-', '{url}', '{name}'],\n                       stdin=jfh)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_drop_after(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.repo.set_gitattributes([('a*', {'annex.largefiles': 'nothing'})])\n        # make some files go to git, so we could test that we do not blow\n        # while trying to drop what is in git not annex\n        res = ds.addurls(self.json_file, '{url}', '{name}', drop_after=True)\n\n        assert_result_count(res, 3, action='addurl', status='ok')  # a, b, c  even if a goes to git\n        assert_result_count(res, 2, action='drop', status='ok')  # b, c\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_from_key_invalid_format(self, path):\n        ds = Dataset(path).create(force=True)\n        for fmt in [\"{name}-which-has-no-double-dash\",\n                    # Invalid hash length.\n                    \"MD5-s{size}--{md5sum}a\",\n                    # Invalid hash content.\n                    \"MD5-s{size}--\" + 32 * \"q\"]:\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           key=fmt, exclude_autometa=\"*\")\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_from_key(self, key_arg, expected_backend, fake_dates,\n                               path):\n        ds = Dataset(path).create(force=True, fake_dates=fake_dates)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        ds.addurls(self.json_file, \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                   key=key_arg)\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], expected_backend)\n            assert_false(pstat[\"has_content\"])\n\n        get_res = ds.get(paths, result_renderer=None, on_failure=\"ignore\")\n        assert_result_count(get_res, 2, action=\"get\", status=\"ok\")\n\n    def test_addurls_from_key(self):\n        fn = self.check_addurls_from_key\n        for case in [\n                (fn, \"MD5-s{size}--{md5sum}\", \"MD5\"),\n                (fn, \"MD5E-s{size}--{md5sum}.dat\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5-s{size}--{md5sum}\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5E-s{size}--{md5sum}.dat\", \"MD5\")]:\n            yield case + (False,)\n            yield case + (True,)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_row_missing_key_fields(self, path):\n        ds = Dataset(path).create(force=True)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        data = self.data.copy()\n        for row in data:\n            if row[\"name\"] == \"b\":\n                del row[\"md5sum\"]\n                break\n        with patch(\"sys.stdin\", new=StringIO(json.dumps(data))):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                       key=\"MD5-s{size}--{md5sum}\")\n\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], \"MD5\")\n            assert_false(pstat[\"has_content\"])",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 419,
                            "end_lineno": 848,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/plugin/tests/test_addurls.py",
                            "type": "method",
                            "name": "TestAddurls.test_addurls_from_list",
                            "content": "class TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.data = [{\"url\": cls.url + \"udir/a.dat\",\n                     \"name\": \"a\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"3fb7c40c70b0ed19da713bd69ee12014\",\n                     \"size\": \"9\"},\n                    {\"url\": cls.url + \"udir/b.dat\",\n                     \"name\": \"b\",\n                     \"subdir\": \"bar\",\n                     \"md5sum\": \"\",\n                     \"size\": \"\"},\n                    {\"url\": cls.url + \"udir/c.dat\",\n                     \"name\": \"c\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"9b72648021b70b8c522642e4490d7ac3\",\n                     \"size\": \"9\"}]\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(cls.data, jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return len(ds.repo.get_revisions(\"git-annex\"))\n\n        n_annex_commits = get_annex_commit_counts()\n\n        # Meanwhile also test that we can specify path relative\n        # to the top of the dataset, as we generally treat paths in\n        # Python API, and it will be the one saved in commit\n        # message record\n        json_file = op.relpath(self.json_file, ds.path)\n\n        ds.addurls(json_file, \"{url}\", \"{name}\",\n                   exclude_autometa=\"(md5sum|size)\")\n        ok_startswith(ds.repo.format_commit('%b', DEFAULT_BRANCH), f\"url_file='{json_file}'\")\n\n        filenames = [\"a\", \"b\", \"c\"]\n        for fname in filenames:\n            ok_exists(op.join(ds.path, fname))\n\n        for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                         [\"foo\", \"bar\", \"foo\"]):\n            assert_dict_equal(meta,\n                              {\"subdir\": [subdir], \"name\": [fname]})\n\n        # Ignore this check if we're faking dates because that disables\n        # batch mode.\n        # Also ignore if on Windows as it seems as if a git-annex bug \n        # leads to separate meta data commits: \n        # https://github.com/datalad/datalad/pull/5202#discussion_r535429704\n        if not (dl_cfg.get('datalad.fake-dates') or on_windows):\n            # We should have two new commits on the git-annex: one for the\n            # added urls and one for the added metadata.\n            eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n        # Add to already existing links, overwriting.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                       ifexists=\"overwrite\")\n            for fname in filenames:\n                assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                          cml.out)\n\n        # Add to already existing links, skipping.\n        assert_in_results(\n            ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n            action=\"addurls\",\n            status=\"notneeded\")\n\n        # Add to already existing links works, as long content is the same.\n        ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n        # But it fails if something has changed.\n        ds.unlock(\"a\")\n        with open(op.join(ds.path, \"a\"), \"w\") as ofh:\n            ofh.write(\"changed\")\n        ds.save(\"a\")\n\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_unbound_dataset(self, path):\n        def check(ds, dataset_arg, url_file, fname_format):\n            subdir = op.join(ds.path, \"subdir\")\n            os.mkdir(subdir)\n            with chpwd(subdir):\n                shutil.copy(self.json_file, \"in.json\")\n                addurls(dataset_arg, url_file, \"{url}\", fname_format)\n                # Files specified in the CSV file are always relative to the\n                # dataset.\n                for fname in [\"a\", \"b\", \"c\"]:\n                    ok_exists(op.join(ds.path, fname))\n\n        # The input file is relative to the current working directory, as\n        # with other commands.\n        ds0 = Dataset(op.join(path, \"ds0\")).create()\n        check(ds0, None, \"in.json\", \"{name}\")\n        # Likewise the input file is relative to the current working directory\n        # if a string dataset argument is given.\n        ds1 = Dataset(op.join(path, \"ds1\")).create()\n        check(ds1, ds1.path, \"in.json\", \"{name}\")\n        # A leading \"./\" doesn't confuse addurls() into downloading the file\n        # into the subdirectory.\n        ds2 = Dataset(op.join(path, \"ds2\")).create()\n        check(ds2, None, \"in.json\", \"./{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\",\n                cfg_proc=[\"yoda\"])\n        for fname in [\"a\", \"b\", \"c\", \"code\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile\n    def test_addurls_from_list(self, path):\n        ds = Dataset(path).create()\n        ds.addurls(self.data, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        for save in True, False:\n            label = \"save\" if save else \"nosave\"\n            ds.addurls(self.json_file, \"{url}\",\n                       \"{subdir}-\" + label + \"//{name}\",\n                       save=save,\n                       cfg_proc=[\"yoda\"])\n\n            subdirs = [op.join(ds.path, \"{}-{}\".format(d, label))\n                       for d in [\"foo\", \"bar\"]]\n            subdir_files = dict(zip(subdirs, [[\"a\", \"c\"], [\"b\"]]))\n\n            for subds, fnames in subdir_files.items():\n                for fname in fnames:\n                    ok_exists(op.join(subds, fname))\n                # cfg_proc was applied generated subdatasets.\n                ok_exists(op.join(subds, \"code\"))\n            if save:\n                assert_repo_status(path)\n            else:\n                # The datasets are create but not saved (since asked not to)\n                assert_repo_status(path, untracked=subdirs)\n                # but the downloaded files aren't.\n                for subds, fnames in subdir_files.items():\n                    assert_repo_status(subds, added=fnames)\n\n        # Now save the \"--nosave\" changes and check that we have\n        # all the subdatasets.\n        ds.save()\n        eq_(set(subdatasets(dataset=ds, recursive=True,\n                            result_xfm=\"relpaths\")),\n            {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n        # We don't try to recreate existing subdatasets.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n            assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with assert_raises(IncompleteResultsError) as raised:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n        assert_in(\"There are file name collisions\", str(raised.exception))\n\n        ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n        for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file,\n                      \"{url}/nofilename/\",\n                      \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata_\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with patch.object(ds.repo, 'set_metadata_', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\",\n            jobs=3)\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n\n    @with_tree({\"in\": \"\"})\n    def test_addurls_invalid_input(self, path):\n        ds = Dataset(path).create(force=True)\n        in_file = op.join(path, \"in\")\n        for in_type in au.INPUT_TYPES:\n            with assert_raises(IncompleteResultsError) as exc:\n                ds.addurls(in_file, \"{url}\", \"{name}\", input_type=in_type)\n            assert_in(\"Failed to read\", str(exc.exception))\n\n    @with_tree({\"in.csv\": \"url,name,subdir\",\n                \"in.tsv\": \"url\\tname\\tsubdir\",\n                \"in.json\": \"[]\"})\n    def test_addurls_no_rows(self, path):\n        ds = Dataset(path).create(force=True)\n        for fname in [\"in.csv\", \"in.tsv\", \"in.json\"]:\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                assert_in_results(\n                    ds.addurls(fname, \"{url}\", \"{name}\"),\n                    action=\"addurls\",\n                    status=\"notneeded\")\n                cml.assert_logged(\"No rows\", regex=False)\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_stdin_input(self, input_text, input_type, path):\n        ds = Dataset(path).create(force=True)\n        with patch(\"sys.stdin\", new=StringIO(input_text)):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", input_type=input_type)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    def test_addurls_stdin_input(self):\n        def make_test(text, input_type, description):\n            def fn():\n                self.check_addurls_stdin_input(json_text, \"ext\")\n            fn.description = description\n            return fn\n\n        with open(self.json_file) as jfh:\n            json_text = jfh.read()\n\n        yield make_test(json_text, \"ext\", \"json,default input type\")\n        yield make_test(json_text, \"json\", \"json,json input type\")\n\n        def make_delim_text(delim):\n            row = \"{name}\" + delim + \"{url}\"\n            return \"\\n\".join(\n                [row.format(name=\"name\", url=\"url\")] +\n                [row.format(**rec) for rec in json.loads(json_text)])\n\n        yield make_test(make_delim_text(\",\"), \"csv\", \"csv,csv input type\")\n        yield make_test(make_delim_text(\"\\t\"), \"tsv\", \"tsv,tsv input type\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_stdin_input_command_line(self, path):\n        # The previous test checks all the cases, but it overrides sys.stdin.\n        # Do a simple check that's closer to a command line call.\n        Dataset(path).create(force=True)\n        runner = WitlessRunner(cwd=path)\n        with open(self.json_file) as jfh:\n            runner.run([\"datalad\", \"addurls\", '-', '{url}', '{name}'],\n                       stdin=jfh)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_drop_after(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.repo.set_gitattributes([('a*', {'annex.largefiles': 'nothing'})])\n        # make some files go to git, so we could test that we do not blow\n        # while trying to drop what is in git not annex\n        res = ds.addurls(self.json_file, '{url}', '{name}', drop_after=True)\n\n        assert_result_count(res, 3, action='addurl', status='ok')  # a, b, c  even if a goes to git\n        assert_result_count(res, 2, action='drop', status='ok')  # b, c\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_from_key_invalid_format(self, path):\n        ds = Dataset(path).create(force=True)\n        for fmt in [\"{name}-which-has-no-double-dash\",\n                    # Invalid hash length.\n                    \"MD5-s{size}--{md5sum}a\",\n                    # Invalid hash content.\n                    \"MD5-s{size}--\" + 32 * \"q\"]:\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           key=fmt, exclude_autometa=\"*\")\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_from_key(self, key_arg, expected_backend, fake_dates,\n                               path):\n        ds = Dataset(path).create(force=True, fake_dates=fake_dates)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        ds.addurls(self.json_file, \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                   key=key_arg)\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], expected_backend)\n            assert_false(pstat[\"has_content\"])\n\n        get_res = ds.get(paths, result_renderer=None, on_failure=\"ignore\")\n        assert_result_count(get_res, 2, action=\"get\", status=\"ok\")\n\n    def test_addurls_from_key(self):\n        fn = self.check_addurls_from_key\n        for case in [\n                (fn, \"MD5-s{size}--{md5sum}\", \"MD5\"),\n                (fn, \"MD5E-s{size}--{md5sum}.dat\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5-s{size}--{md5sum}\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5E-s{size}--{md5sum}.dat\", \"MD5\")]:\n            yield case + (False,)\n            yield case + (True,)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_row_missing_key_fields(self, path):\n        ds = Dataset(path).create(force=True)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        data = self.data.copy()\n        for row in data:\n            if row[\"name\"] == \"b\":\n                del row[\"md5sum\"]\n                break\n        with patch(\"sys.stdin\", new=StringIO(json.dumps(data))):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                       key=\"MD5-s{size}--{md5sum}\")\n\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], \"MD5\")\n            assert_false(pstat[\"has_content\"])",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 558,
                            "end_lineno": 562,
                            "parent": {
                                "file_name": "datalad/plugin/tests/test_addurls.py",
                                "type": "class",
                                "name": "TestAddurls",
                                "content": "class TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.data = [{\"url\": cls.url + \"udir/a.dat\",\n                     \"name\": \"a\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"3fb7c40c70b0ed19da713bd69ee12014\",\n                     \"size\": \"9\"},\n                    {\"url\": cls.url + \"udir/b.dat\",\n                     \"name\": \"b\",\n                     \"subdir\": \"bar\",\n                     \"md5sum\": \"\",\n                     \"size\": \"\"},\n                    {\"url\": cls.url + \"udir/c.dat\",\n                     \"name\": \"c\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"9b72648021b70b8c522642e4490d7ac3\",\n                     \"size\": \"9\"}]\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(cls.data, jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return len(ds.repo.get_revisions(\"git-annex\"))\n\n        n_annex_commits = get_annex_commit_counts()\n\n        # Meanwhile also test that we can specify path relative\n        # to the top of the dataset, as we generally treat paths in\n        # Python API, and it will be the one saved in commit\n        # message record\n        json_file = op.relpath(self.json_file, ds.path)\n\n        ds.addurls(json_file, \"{url}\", \"{name}\",\n                   exclude_autometa=\"(md5sum|size)\")\n        ok_startswith(ds.repo.format_commit('%b', DEFAULT_BRANCH), f\"url_file='{json_file}'\")\n\n        filenames = [\"a\", \"b\", \"c\"]\n        for fname in filenames:\n            ok_exists(op.join(ds.path, fname))\n\n        for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                         [\"foo\", \"bar\", \"foo\"]):\n            assert_dict_equal(meta,\n                              {\"subdir\": [subdir], \"name\": [fname]})\n\n        # Ignore this check if we're faking dates because that disables\n        # batch mode.\n        # Also ignore if on Windows as it seems as if a git-annex bug \n        # leads to separate meta data commits: \n        # https://github.com/datalad/datalad/pull/5202#discussion_r535429704\n        if not (dl_cfg.get('datalad.fake-dates') or on_windows):\n            # We should have two new commits on the git-annex: one for the\n            # added urls and one for the added metadata.\n            eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n        # Add to already existing links, overwriting.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                       ifexists=\"overwrite\")\n            for fname in filenames:\n                assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                          cml.out)\n\n        # Add to already existing links, skipping.\n        assert_in_results(\n            ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n            action=\"addurls\",\n            status=\"notneeded\")\n\n        # Add to already existing links works, as long content is the same.\n        ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n        # But it fails if something has changed.\n        ds.unlock(\"a\")\n        with open(op.join(ds.path, \"a\"), \"w\") as ofh:\n            ofh.write(\"changed\")\n        ds.save(\"a\")\n\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_unbound_dataset(self, path):\n        def check(ds, dataset_arg, url_file, fname_format):\n            subdir = op.join(ds.path, \"subdir\")\n            os.mkdir(subdir)\n            with chpwd(subdir):\n                shutil.copy(self.json_file, \"in.json\")\n                addurls(dataset_arg, url_file, \"{url}\", fname_format)\n                # Files specified in the CSV file are always relative to the\n                # dataset.\n                for fname in [\"a\", \"b\", \"c\"]:\n                    ok_exists(op.join(ds.path, fname))\n\n        # The input file is relative to the current working directory, as\n        # with other commands.\n        ds0 = Dataset(op.join(path, \"ds0\")).create()\n        check(ds0, None, \"in.json\", \"{name}\")\n        # Likewise the input file is relative to the current working directory\n        # if a string dataset argument is given.\n        ds1 = Dataset(op.join(path, \"ds1\")).create()\n        check(ds1, ds1.path, \"in.json\", \"{name}\")\n        # A leading \"./\" doesn't confuse addurls() into downloading the file\n        # into the subdirectory.\n        ds2 = Dataset(op.join(path, \"ds2\")).create()\n        check(ds2, None, \"in.json\", \"./{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\",\n                cfg_proc=[\"yoda\"])\n        for fname in [\"a\", \"b\", \"c\", \"code\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile\n    def test_addurls_from_list(self, path):\n        ds = Dataset(path).create()\n        ds.addurls(self.data, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        for save in True, False:\n            label = \"save\" if save else \"nosave\"\n            ds.addurls(self.json_file, \"{url}\",\n                       \"{subdir}-\" + label + \"//{name}\",\n                       save=save,\n                       cfg_proc=[\"yoda\"])\n\n            subdirs = [op.join(ds.path, \"{}-{}\".format(d, label))\n                       for d in [\"foo\", \"bar\"]]\n            subdir_files = dict(zip(subdirs, [[\"a\", \"c\"], [\"b\"]]))\n\n            for subds, fnames in subdir_files.items():\n                for fname in fnames:\n                    ok_exists(op.join(subds, fname))\n                # cfg_proc was applied generated subdatasets.\n                ok_exists(op.join(subds, \"code\"))\n            if save:\n                assert_repo_status(path)\n            else:\n                # The datasets are create but not saved (since asked not to)\n                assert_repo_status(path, untracked=subdirs)\n                # but the downloaded files aren't.\n                for subds, fnames in subdir_files.items():\n                    assert_repo_status(subds, added=fnames)\n\n        # Now save the \"--nosave\" changes and check that we have\n        # all the subdatasets.\n        ds.save()\n        eq_(set(subdatasets(dataset=ds, recursive=True,\n                            result_xfm=\"relpaths\")),\n            {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n        # We don't try to recreate existing subdatasets.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n            assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with assert_raises(IncompleteResultsError) as raised:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n        assert_in(\"There are file name collisions\", str(raised.exception))\n\n        ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n        for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file,\n                      \"{url}/nofilename/\",\n                      \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata_\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with patch.object(ds.repo, 'set_metadata_', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\",\n            jobs=3)\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n\n    @with_tree({\"in\": \"\"})\n    def test_addurls_invalid_input(self, path):\n        ds = Dataset(path).create(force=True)\n        in_file = op.join(path, \"in\")\n        for in_type in au.INPUT_TYPES:\n            with assert_raises(IncompleteResultsError) as exc:\n                ds.addurls(in_file, \"{url}\", \"{name}\", input_type=in_type)\n            assert_in(\"Failed to read\", str(exc.exception))\n\n    @with_tree({\"in.csv\": \"url,name,subdir\",\n                \"in.tsv\": \"url\\tname\\tsubdir\",\n                \"in.json\": \"[]\"})\n    def test_addurls_no_rows(self, path):\n        ds = Dataset(path).create(force=True)\n        for fname in [\"in.csv\", \"in.tsv\", \"in.json\"]:\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                assert_in_results(\n                    ds.addurls(fname, \"{url}\", \"{name}\"),\n                    action=\"addurls\",\n                    status=\"notneeded\")\n                cml.assert_logged(\"No rows\", regex=False)\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_stdin_input(self, input_text, input_type, path):\n        ds = Dataset(path).create(force=True)\n        with patch(\"sys.stdin\", new=StringIO(input_text)):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", input_type=input_type)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    def test_addurls_stdin_input(self):\n        def make_test(text, input_type, description):\n            def fn():\n                self.check_addurls_stdin_input(json_text, \"ext\")\n            fn.description = description\n            return fn\n\n        with open(self.json_file) as jfh:\n            json_text = jfh.read()\n\n        yield make_test(json_text, \"ext\", \"json,default input type\")\n        yield make_test(json_text, \"json\", \"json,json input type\")\n\n        def make_delim_text(delim):\n            row = \"{name}\" + delim + \"{url}\"\n            return \"\\n\".join(\n                [row.format(name=\"name\", url=\"url\")] +\n                [row.format(**rec) for rec in json.loads(json_text)])\n\n        yield make_test(make_delim_text(\",\"), \"csv\", \"csv,csv input type\")\n        yield make_test(make_delim_text(\"\\t\"), \"tsv\", \"tsv,tsv input type\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_stdin_input_command_line(self, path):\n        # The previous test checks all the cases, but it overrides sys.stdin.\n        # Do a simple check that's closer to a command line call.\n        Dataset(path).create(force=True)\n        runner = WitlessRunner(cwd=path)\n        with open(self.json_file) as jfh:\n            runner.run([\"datalad\", \"addurls\", '-', '{url}', '{name}'],\n                       stdin=jfh)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_drop_after(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.repo.set_gitattributes([('a*', {'annex.largefiles': 'nothing'})])\n        # make some files go to git, so we could test that we do not blow\n        # while trying to drop what is in git not annex\n        res = ds.addurls(self.json_file, '{url}', '{name}', drop_after=True)\n\n        assert_result_count(res, 3, action='addurl', status='ok')  # a, b, c  even if a goes to git\n        assert_result_count(res, 2, action='drop', status='ok')  # b, c\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_from_key_invalid_format(self, path):\n        ds = Dataset(path).create(force=True)\n        for fmt in [\"{name}-which-has-no-double-dash\",\n                    # Invalid hash length.\n                    \"MD5-s{size}--{md5sum}a\",\n                    # Invalid hash content.\n                    \"MD5-s{size}--\" + 32 * \"q\"]:\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           key=fmt, exclude_autometa=\"*\")\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_from_key(self, key_arg, expected_backend, fake_dates,\n                               path):\n        ds = Dataset(path).create(force=True, fake_dates=fake_dates)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        ds.addurls(self.json_file, \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                   key=key_arg)\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], expected_backend)\n            assert_false(pstat[\"has_content\"])\n\n        get_res = ds.get(paths, result_renderer=None, on_failure=\"ignore\")\n        assert_result_count(get_res, 2, action=\"get\", status=\"ok\")\n\n    def test_addurls_from_key(self):\n        fn = self.check_addurls_from_key\n        for case in [\n                (fn, \"MD5-s{size}--{md5sum}\", \"MD5\"),\n                (fn, \"MD5E-s{size}--{md5sum}.dat\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5-s{size}--{md5sum}\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5E-s{size}--{md5sum}.dat\", \"MD5\")]:\n            yield case + (False,)\n            yield case + (True,)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_row_missing_key_fields(self, path):\n        ds = Dataset(path).create(force=True)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        data = self.data.copy()\n        for row in data:\n            if row[\"name\"] == \"b\":\n                del row[\"md5sum\"]\n                break\n        with patch(\"sys.stdin\", new=StringIO(json.dumps(data))):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                       key=\"MD5-s{size}--{md5sum}\")\n\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], \"MD5\")\n            assert_false(pstat[\"has_content\"])",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 419,
                                "end_lineno": 848,
                                "parent": null
                            }
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "2f8ec83b44108d3b022c778a99d648105028f202^",
    "new_commit_hash": "2f8ec83b44108d3b022c778a99d648105028f202",
    "commit_message": "Merge pull request #5285 from kyleam/addurls-no-file\n\nNF: addurls: Support passing list of records",
    "commit_date": "2021-01-09T10:33:40+01:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "2f8ec83b44108d3b022c778a99d648105028f202",
    "test_file_codes": [
        "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# -*- coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test addurls plugin\"\"\"\n\nimport json\nimport logging\nimport os\nimport os.path as op\nimport shutil\nimport tempfile\n\nfrom unittest.mock import patch\n\nfrom io import StringIO\n\nfrom datalad.api import addurls, Dataset, subdatasets\nfrom datalad.cmd import WitlessRunner\nfrom datalad.consts import WEB_SPECIAL_REMOTE_UUID\nimport datalad.plugin.addurls as au\nfrom datalad.support.exceptions import IncompleteResultsError\nfrom datalad.support.external_versions import external_versions\nfrom datalad.tests.utils import (\n    assert_dict_equal,\n    assert_false,\n    assert_in,\n    assert_in_results,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    assert_repo_status,\n    assert_result_count,\n    assert_true,\n    chpwd,\n    create_tree,\n    eq_,\n    HTTPPath,\n    known_failure_githubci_win,\n    ok_exists,\n    ok_startswith,\n    skip_if,\n    SkipTest,\n    swallow_logs,\n    with_tempfile,\n    with_tree,\n    on_windows,\n    DEFAULT_BRANCH,\n)\nfrom datalad.utils import get_tempfile_kwargs, rmtemp\nfrom datalad import cfg as dl_cfg\n\n\ndef test_formatter():\n    idx_to_name = {i: \"col{}\".format(i) for i in range(4)}\n    values = {\"col{}\".format(i): \"value{}\".format(i) for i in range(4)}\n\n    fmt = au.Formatter(idx_to_name)\n\n    eq_(fmt.format(\"{0}\", values), \"value0\")\n    eq_(fmt.format(\"{0}\", values), fmt.format(\"{col0}\", values))\n\n    # Integer placeholders outside of `idx_to_name` don't work.\n    assert_raises(KeyError, fmt.format, \"{4}\", values, 1, 2, 3, 4)\n\n    # If the named placeholder is not in `values`, falls back to normal\n    # formatting.\n    eq_(fmt.format(\"{notinvals}\", values, notinvals=\"ok\"), \"ok\")\n\n\ndef test_formatter_lower_case():\n    fmt = au.Formatter({0: \"key\"})\n    eq_(fmt.format(\"{key!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{0!l}\", {\"key\": \"UP\"}), \"up\")\n    eq_(fmt.format(\"{other!s}\", {}, other=[1, 2]), \"[1, 2]\")\n\n\ndef test_formatter_no_idx_map():\n    fmt = au.Formatter({})\n    assert_raises(KeyError, fmt.format, \"{0}\", {\"col0\": \"value0\"})\n\n\ndef test_formatter_no_mapping_arg():\n    fmt = au.Formatter({})\n    assert_raises(ValueError, fmt.format, \"{0}\", \"not a mapping\")\n\n\ndef test_formatter_placeholder_with_spaces():\n    fmt = au.Formatter({})\n    eq_(fmt.format(\"{with spaces}\", {\"with spaces\": \"value0\"}), \"value0\")\n\n\ndef test_formatter_placeholder_nonpermitted_chars():\n    fmt = au.Formatter({})\n\n    # Can't assess keys with !, which will be interpreted as a conversion flag.\n    eq_(fmt.format(\"{key!r}\", {\"key!r\": \"value0\"}, key=\"x\"), \"'x'\")\n    assert_raises(KeyError,\n                  fmt.format, \"{key!r}\", {\"key!r\": \"value0\"})\n\n    # Same for \":\".\n    eq_(fmt.format(\"{key:<5}\", {\"key:<5\": \"value0\"}, key=\"x\"), \"x    \")\n    assert_raises(KeyError,\n                  fmt.format, \"{key:<5}\", {\"key:<5\": \"value0\"})\n\n\ndef test_formatter_missing_arg():\n    fmt = au.Formatter({}, \"NA\")\n    eq_(fmt.format(\"{here},{nothere}\", {\"here\": \"ok\", \"nothere\": \"\"}),\n        \"ok,NA\")\n\n\ndef test_repformatter():\n    fmt = au.RepFormatter({})\n\n    for i in range(3):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"x\"}), \"x{}\".format(i))\n    # A new result gets a fresh index.\n    for i in range(2):\n        eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"y\"}), \"y{}\".format(i))\n    # We count even if _repindex isn't there.\n    eq_(fmt.format(\"{c}\", {\"c\": \"z0\"}), \"z0\")\n    eq_(fmt.format(\"{c}{_repindex}\", {\"c\": \"z\"}), \"z1\")\n\n\ndef test_clean_meta_args():\n    for args, expect in [([\"field=\"], {}),\n                         ([\" field=yes \"], {\"field\": \"yes\"}),\n                         ([\"field= value=\"], {\"field\": \"value=\"})]:\n        eq_(au.clean_meta_args(args), expect)\n\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"noequal\"])\n    assert_raises(ValueError,\n                  au.clean_meta_args,\n                  [\"=value\"])\n\n\ndef test_get_subpaths():\n    for fname, expect in [\n            (op.join(\"no\", \"dbl\", \"slash\"),\n             (op.join(\"no\", \"dbl\", \"slash\"), [])),\n            (\"p1//n\",\n             (op.join(\"p1\", \"n\"), [\"p1\"])),\n            (op.join(\"p1//p2\", \"p3//n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"n\"),\n              [\"p1\", op.join(\"p1\", \"p2\", \"p3\")])),\n            (op.join(\"p1//p2\", \"p3//p4\", \"p5//\", \"n\"),\n             (op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\", \"n\"),\n              [\"p1\",\n               op.join(\"p1\", \"p2\", \"p3\"),\n               op.join(\"p1\", \"p2\", \"p3\", \"p4\", \"p5\")])),\n            (\"//n\", (op.sep + \"n\", [\"\"])),\n            (\"n//\", (\"n\" + op.sep, [\"n\"]))]:\n        eq_(au.get_subpaths(fname), expect)\n\n\ndef test_sort_paths():\n    paths = [op.join(\"x\", \"a\", \"b\"),\n             \"z\",\n             op.join(\"y\", \"b\"),\n             op.join(\"y\", \"a\")]\n    expected = [\"z\",\n                op.join(\"y\", \"a\"),\n                op.join(\"y\", \"b\"),\n                op.join(\"x\", \"a\", \"b\")]\n    eq_(list(au.sort_paths(paths)), expected)\n\n\ndef test_is_legal_metafield():\n    for legal in [\"legal\", \"0\", \"legal_\"]:\n        assert_true(au.is_legal_metafield(legal))\n    for notlegal in [\"_not\", \"with space\"]:\n        assert_false(au.is_legal_metafield(notlegal))\n\n\ndef test_filter_legal_metafield():\n    eq_(au.filter_legal_metafield([\"legal\", \"_not\", \"legal_still\"]),\n        [\"legal\", \"legal_still\"])\n\n\ndef test_fmt_to_name():\n    eq_(au.fmt_to_name(\"{name}\", {}), \"name\")\n    eq_(au.fmt_to_name(\"{0}\", {0: \"name\"}), \"name\")\n    eq_(au.fmt_to_name(\"{1}\", {0: \"name\"}), \"1\")\n\n    assert_false(au.fmt_to_name(\"frontmatter{name}\", {}))\n    assert_false(au.fmt_to_name(\"{name}backmatter\", {}))\n    assert_false(au.fmt_to_name(\"{two}{names}\", {}))\n    assert_false(au.fmt_to_name(\"\", {}))\n    assert_false(au.fmt_to_name(\"nonames\", {}))\n    assert_false(au.fmt_to_name(\"{}\", {}))\n\n\ndef test_get_file_parts():\n    assert_dict_equal(au.get_file_parts(\"file.tar.gz\", \"prefix\"),\n                      {\"prefix\": \"file.tar.gz\",\n                       \"prefix_root_py\": \"file.tar\",\n                       \"prefix_ext_py\": \".gz\",\n                       \"prefix_root\": \"file\",\n                       \"prefix_ext\": \".tar.gz\"})\n\n\ndef test_get_url_parts():\n    eq_(au.get_url_parts(\"\"), {})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org\"),\n                      {\"_url_hostname\": \"datalad.org\"})\n\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      {\"_url_hostname\": \"datalad.org\",\n                       \"_url0\": \"about.html\",\n                       \"_url_basename\": \"about.html\",\n                       \"_url_basename_root_py\": \"about\",\n                       \"_url_basename_ext_py\": \".html\",\n                       \"_url_basename_root\": \"about\",\n                       \"_url_basename_ext\": \".html\"})\n    assert_dict_equal(au.get_url_parts(\"http://datalad.org/about.html\"),\n                      au.get_url_parts(\"http://datalad.org//about.html\"))\n\n    assert_dict_equal(\n        au.get_url_parts(\"http://datalad.org/for/git-users\"),\n        {\"_url_hostname\": \"datalad.org\",\n         \"_url0\": \"for\",\n         \"_url1\": \"git-users\",\n         \"_url_basename\": \"git-users\",\n         \"_url_basename_root_py\": \"git-users\",\n         \"_url_basename_ext_py\": \"\",\n         \"_url_basename_root\": \"git-users\",\n         \"_url_basename_ext\": \"\"})\n\n\nST_DATA = {\"header\": [\"name\", \"debut_season\", \"age_group\", \"now_dead\"],\n           \"rows\": [{\"name\": \"will\", \"debut_season\": 1,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"},\n                    {\"name\": \"bob\", \"debut_season\": 2,\n                     \"age_group\": \"adult\", \"now_dead\": \"yes\"},\n                    {\"name\": \"scott\", \"debut_season\": 1,\n                     \"age_group\": \"adult\", \"now_dead\": \"no\"},\n                    {\"name\": \"max\", \"debut_season\": 2,\n                     \"age_group\": \"kid\", \"now_dead\": \"no\"}]}\n\n\ndef json_stream(data):\n    stream = StringIO()\n    json.dump(data, stream)\n    stream.seek(0)\n    return stream\n\n\n@known_failure_githubci_win\ndef test_extract():\n    info, subpaths = au.extract(\n        ST_DATA[\"rows\"],\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\")\n\n    eq_(subpaths,\n        [\"adult\", \"kid\", op.join(\"adult\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"kid\", \"no\")])\n\n    eq_([d[\"url\"] for d in info],\n        [\"will_1.com\", \"bob_2.com\", \"scott_1.com\", \"max_2.com\"])\n\n    eq_([d[\"filename\"] for d in info],\n        [op.join(\"kid\", \"no\", \"will.csv\"), op.join(\"adult\", \"yes\", \"bob.csv\"),\n         op.join(\"adult\", \"no\", \"scott.csv\"), op.join(\"kid\", \"no\", \"max.csv\")])\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\", \"debut_season\": \"2\",\n                \"now_dead\": \"yes\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\", \"debut_season\": \"1\",\n                \"now_dead\": \"no\"},\n               {\"name\": \"max\", \"age_group\": \"kid\", \"debut_season\": \"2\",\n                \"now_dead\": \"no\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n    eq_([d[\"subpath\"] for d in info],\n        [op.join(\"kid\", \"no\"), op.join(\"adult\", \"yes\"), op.join(\"adult\", \"no\"), op.join(\"kid\", \"no\")])\n\n\ndef test_extract_disable_autometa():\n    info, _ = au.extract(\n        ST_DATA[\"rows\"],\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"*\",\n        meta=[\"group={age_group}\"])\n\n    eq_([d[\"meta_args\"] for d in info],\n        [{\"group\": \"kid\"}, {\"group\": \"adult\"}, {\"group\": \"adult\"},\n         {\"group\": \"kid\"}])\n\n\ndef test_extract_exclude_autometa_regexp():\n    info, _ = au.extract(\n        ST_DATA[\"rows\"],\n        url_format=\"{name}_{debut_season}.com\",\n        filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n        exclude_autometa=\"ea\")\n\n    expects = [{\"name\": \"will\", \"age_group\": \"kid\"},\n               {\"name\": \"bob\", \"age_group\": \"adult\"},\n               {\"name\": \"scott\", \"age_group\": \"adult\"},\n               {\"name\": \"max\", \"age_group\": \"kid\"}]\n    for d, expect in zip(info, expects):\n        assert_dict_equal(d[\"meta_args\"], expect)\n\n\ndef check_extract_csv_json_equal(input_type):\n    delim = \"\\t\" if input_type == \"tsv\" else \",\"\n\n    keys = ST_DATA[\"header\"]\n    csv_rows = [delim.join(keys)]\n    csv_rows.extend(delim.join(str(row[k]) for k in keys)\n                    for row in ST_DATA[\"rows\"])\n\n    kwds = dict(filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n                url_format=\"{name}_{debut_season}.com\",\n                meta=[\"group={age_group}\"])\n\n    json_output = au.extract(\n        *au._read(json_stream(ST_DATA[\"rows\"]), \"json\"), **kwds)\n    csv_output = au.extract(\n        *au._read(csv_rows, input_type), **kwds)\n\n    eq_(json_output, csv_output)\n\n\ndef test_extract_csv_tsv_json_equal():\n    yield check_extract_csv_json_equal, \"csv\"\n    yield check_extract_csv_json_equal, \"tsv\"\n\n\ndef test_extract_wrong_input_type():\n    assert_raises(ValueError,\n                  au._read, None, \"invalid_input_type\")\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_nonannex_repo(path):\n    ds = Dataset(path).create(force=True, annex=False)\n    with assert_raises(IncompleteResultsError) as raised:\n        ds.addurls(\"dummy_arg0\", \"dummy_arg1\", \"dummy_arg2\")\n    assert_in(\"not an annex repo\", str(raised.exception))\n\n\n@with_tree({\"in.csv\": \"linky,abcd\\nhttps://datalad.org,f\"})\ndef test_addurls_unknown_placeholder(path):\n    ds = Dataset(path).create(force=True)\n    # Close but wrong URL placeholder\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{link}\", \"{abcd}\", dry_run=True)\n    assert_in(\"linky\", str(exc.exception))\n    # Close but wrong file name placeholder\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{abc}\", dry_run=True)\n    assert_in(\"abcd\", str(exc.exception))\n    # Out-of-bounds index.\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{3}\", dry_run=True)\n    assert_in(\"index\", str(exc.exception))\n\n    # Suggestions also work for automatic file name placeholders\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{_url_hostnam}\", dry_run=True)\n    assert_in(\"_url_hostname\", str(exc.exception))\n    # ... though if you whiff on the beginning prefix, we don't suggest\n    # anything because we decide to generate those fields based on detecting\n    # the prefix.\n    with assert_raises(IncompleteResultsError) as exc:\n        ds.addurls(\"in.csv\", \"{linky}\", \"{_uurl_hostnam}\", dry_run=True)\n    assert_not_in(\"_url_hostname\", str(exc.exception))\n\n\n@with_tempfile(mkdir=True)\ndef test_addurls_dry_run(path):\n    ds = Dataset(path).create(force=True)\n\n    json_file = \"links.json\"\n    with open(op.join(ds.path, json_file), \"w\") as jfh:\n        json.dump([{\"url\": \"URL/a.dat\", \"name\": \"a\", \"subdir\": \"foo\"},\n                   {\"url\": \"URL/b.dat\", \"name\": \"b\", \"subdir\": \"bar\"},\n                   {\"url\": \"URL/c.dat\", \"name\": \"c\", \"subdir\": \"foo\"}],\n                  jfh)\n\n    ds.save(message=\"setup\")\n\n    with swallow_logs(new_level=logging.INFO) as cml:\n        ds.addurls(json_file,\n                   \"{url}\",\n                   \"{subdir}//{_url_filename_root}\",\n                   dry_run=True)\n\n        for dir_ in [\"foo\", \"bar\"]:\n            assert_in(\"Would create a subdataset at {}\".format(dir_),\n                      cml.out)\n        assert_in(\n            \"Would download URL/a.dat to {}\".format(\n                os.path.join(path, \"foo\", \"BASE\")),\n            cml.out)\n\n        assert_in(\"Metadata: {}\".format([u\"name=a\", u\"subdir=foo\"]),\n                  cml.out)\n\n\nOLD_EXAMINEKEY = external_versions[\"cmd:annex\"] < \"8.20201116\"\nskip_key_tests = skip_if(\n    OLD_EXAMINEKEY,\n    \"git-annex version does not support `examinekey --migrate-to-backend`\")\n\n\nclass TestAddurls(object):\n\n    @classmethod\n    def setup_class(cls):\n        mktmp_kws = get_tempfile_kwargs()\n        path = tempfile.mkdtemp(**mktmp_kws)\n        create_tree(path,\n                    {\"udir\": {x + \".dat\" + ver: x + \" content\"\n                              for x in \"abcd\"\n                              for ver in [\"\", \".v1\"]}})\n\n        cls._hpath = HTTPPath(path)\n        cls._hpath.start()\n        cls.url = cls._hpath.url\n\n        cls.data = [{\"url\": cls.url + \"udir/a.dat\",\n                     \"name\": \"a\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"3fb7c40c70b0ed19da713bd69ee12014\",\n                     \"size\": \"9\"},\n                    {\"url\": cls.url + \"udir/b.dat\",\n                     \"name\": \"b\",\n                     \"subdir\": \"bar\",\n                     \"md5sum\": \"\",\n                     \"size\": \"\"},\n                    {\"url\": cls.url + \"udir/c.dat\",\n                     \"name\": \"c\",\n                     \"subdir\": \"foo\",\n                     \"md5sum\": \"9b72648021b70b8c522642e4490d7ac3\",\n                     \"size\": \"9\"}]\n        cls.json_file = tempfile.mktemp(suffix=\".json\", **mktmp_kws)\n        with open(cls.json_file, \"w\") as jfh:\n            json.dump(cls.data, jfh)\n\n    @classmethod\n    def teardown_class(cls):\n        cls._hpath.stop()\n        rmtemp(cls._hpath.path)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def get_annex_commit_counts():\n            return len(ds.repo.get_revisions(\"git-annex\"))\n\n        n_annex_commits = get_annex_commit_counts()\n\n        # Meanwhile also test that we can specify path relative\n        # to the top of the dataset, as we generally treat paths in\n        # Python API, and it will be the one saved in commit\n        # message record\n        json_file = op.relpath(self.json_file, ds.path)\n\n        ds.addurls(json_file, \"{url}\", \"{name}\",\n                   exclude_autometa=\"(md5sum|size)\")\n        ok_startswith(ds.repo.format_commit('%b', DEFAULT_BRANCH), f\"url_file='{json_file}'\")\n\n        filenames = [\"a\", \"b\", \"c\"]\n        for fname in filenames:\n            ok_exists(op.join(ds.path, fname))\n\n        for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),\n                                         [\"foo\", \"bar\", \"foo\"]):\n            assert_dict_equal(meta,\n                              {\"subdir\": [subdir], \"name\": [fname]})\n\n        # Ignore this check if we're faking dates because that disables\n        # batch mode.\n        # Also ignore if on Windows as it seems as if a git-annex bug \n        # leads to separate meta data commits: \n        # https://github.com/datalad/datalad/pull/5202#discussion_r535429704\n        if not (dl_cfg.get('datalad.fake-dates') or on_windows):\n            # We should have two new commits on the git-annex: one for the\n            # added urls and one for the added metadata.\n            eq_(n_annex_commits + 2, get_annex_commit_counts())\n\n        # Add to already existing links, overwriting.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                       ifexists=\"overwrite\")\n            for fname in filenames:\n                assert_in(\"Removing {}\".format(os.path.join(path, fname)),\n                          cml.out)\n\n        # Add to already existing links, skipping.\n        assert_in_results(\n            ds.addurls(self.json_file, \"{url}\", \"{name}\", ifexists=\"skip\"),\n            action=\"addurls\",\n            status=\"notneeded\")\n\n        # Add to already existing links works, as long content is the same.\n        ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n        # But it fails if something has changed.\n        ds.unlock(\"a\")\n        with open(op.join(ds.path, \"a\"), \"w\") as ofh:\n            ofh.write(\"changed\")\n        ds.save(\"a\")\n\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_unbound_dataset(self, path):\n        def check(ds, dataset_arg, url_file, fname_format):\n            subdir = op.join(ds.path, \"subdir\")\n            os.mkdir(subdir)\n            with chpwd(subdir):\n                shutil.copy(self.json_file, \"in.json\")\n                addurls(dataset_arg, url_file, \"{url}\", fname_format)\n                # Files specified in the CSV file are always relative to the\n                # dataset.\n                for fname in [\"a\", \"b\", \"c\"]:\n                    ok_exists(op.join(ds.path, fname))\n\n        # The input file is relative to the current working directory, as\n        # with other commands.\n        ds0 = Dataset(op.join(path, \"ds0\")).create()\n        check(ds0, None, \"in.json\", \"{name}\")\n        # Likewise the input file is relative to the current working directory\n        # if a string dataset argument is given.\n        ds1 = Dataset(op.join(path, \"ds1\")).create()\n        check(ds1, ds1.path, \"in.json\", \"{name}\")\n        # A leading \"./\" doesn't confuse addurls() into downloading the file\n        # into the subdirectory.\n        ds2 = Dataset(op.join(path, \"ds2\")).create()\n        check(ds2, None, \"in.json\", \"./{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_create_newdataset(self, path):\n        dspath = os.path.join(path, \"ds\")\n        addurls(dspath, self.json_file, \"{url}\", \"{name}\",\n                cfg_proc=[\"yoda\"])\n        for fname in [\"a\", \"b\", \"c\", \"code\"]:\n            ok_exists(os.path.join(dspath, fname))\n\n    @with_tempfile\n    def test_addurls_from_list(self, path):\n        ds = Dataset(path).create()\n        ds.addurls(self.data, \"{url}\", \"{name}\")\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_subdataset(self, path):\n        ds = Dataset(path).create(force=True)\n\n        for save in True, False:\n            label = \"save\" if save else \"nosave\"\n            ds.addurls(self.json_file, \"{url}\",\n                       \"{subdir}-\" + label + \"//{name}\",\n                       save=save,\n                       cfg_proc=[\"yoda\"])\n\n            subdirs = [op.join(ds.path, \"{}-{}\".format(d, label))\n                       for d in [\"foo\", \"bar\"]]\n            subdir_files = dict(zip(subdirs, [[\"a\", \"c\"], [\"b\"]]))\n\n            for subds, fnames in subdir_files.items():\n                for fname in fnames:\n                    ok_exists(op.join(subds, fname))\n                # cfg_proc was applied generated subdatasets.\n                ok_exists(op.join(subds, \"code\"))\n            if save:\n                assert_repo_status(path)\n            else:\n                # The datasets are create but not saved (since asked not to)\n                assert_repo_status(path, untracked=subdirs)\n                # but the downloaded files aren't.\n                for subds, fnames in subdir_files.items():\n                    assert_repo_status(subds, added=fnames)\n\n        # Now save the \"--nosave\" changes and check that we have\n        # all the subdatasets.\n        ds.save()\n        eq_(set(subdatasets(dataset=ds, recursive=True,\n                            result_xfm=\"relpaths\")),\n            {\"foo-save\", \"bar-save\", \"foo-nosave\", \"bar-nosave\"})\n\n        # We don't try to recreate existing subdatasets.\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}-nosave//{name}\")\n            assert_in(\"Not creating subdataset at existing path\", cml.out)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_repindex(self, path):\n        ds = Dataset(path).create(force=True)\n\n        with assert_raises(IncompleteResultsError) as raised:\n            ds.addurls(self.json_file, \"{url}\", \"{subdir}\")\n        assert_in(\"There are file name collisions\", str(raised.exception))\n\n        ds.addurls(self.json_file, \"{url}\", \"{subdir}-{_repindex}\")\n\n        for fname in [\"foo-0\", \"bar-0\", \"foo-1\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_parts(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_basename}\")\n\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(self.json_file, \"{url}\", \"{_url0}/{_url_filename}\")\n        for fname in [\"a.dat\", \"b.dat\", \"c.dat\"]:\n            ok_exists(op.join(ds.path, \"udir\", fname))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_url_filename_fail(self, path):\n        ds = Dataset(path).create(force=True)\n        assert_raises(IncompleteResultsError,\n                      ds.addurls,\n                      self.json_file,\n                      \"{url}/nofilename/\",\n                      \"{_url0}/{_url_filename}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_metafail(self, path):\n        ds = Dataset(path).create(force=True)\n\n        # Force failure by passing a non-existent file name to annex.\n        fn = ds.repo.set_metadata_\n\n        def set_meta(_, **kwargs):\n            for i in fn(\"wreaking-havoc-and-such\", **kwargs):\n                yield i\n\n        with patch.object(ds.repo, 'set_metadata_', set_meta):\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_dropped_urls(self, path):\n        ds = Dataset(path).create(force=True)\n        with swallow_logs(new_level=logging.WARNING) as cml:\n            ds.addurls(self.json_file, \"\", \"{subdir}//{name}\")\n            assert_re_in(r\".*Dropped [0-9]+ row\\(s\\) that had an empty URL\",\n                         str(cml.out))\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_version(self, path):\n        ds = Dataset(path).create(force=True)\n\n        def version_fn(url):\n            if url.endswith(\"b.dat\"):\n                raise ValueError(\"Scheme error\")\n            return url + \".v1\"\n\n        with patch(\"datalad.plugin.addurls.get_versioned_url\", version_fn):\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           version_urls=True)\n                assert_in(\"b.dat\", str(cml.out))\n\n        names = [\"a\", \"c\"]\n        for fname in names:\n            ok_exists(os.path.join(path, fname))\n\n        whereis = ds.repo.whereis(names, output=\"full\")\n        for fname, info in whereis.items():\n            eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],\n                [\"{}udir/{}.dat.v1\".format(self.url, fname)])\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_deeper(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.addurls(\n            self.json_file, \"{url}\",\n            \"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}\",\n            jobs=3)\n        eq_(set(ds.subdatasets(recursive=True, result_xfm=\"relpaths\")),\n            {\"foo\",\n             \"bar\",\n             op.join(\"foo\", \"adir\", \"foo-again\"),\n             op.join(\"bar\", \"adir\", \"bar-again\"),\n             op.join(\"foo\", \"adir\", \"foo-again\", \"other-ds\"),\n             op.join(\"bar\", \"adir\", \"bar-again\", \"other-ds\")})\n        ok_exists(os.path.join(\n            ds.path, \"foo\", \"adir\", \"foo-again\", \"other-ds\", \"bdir\", \"a\"))\n\n    @with_tree({\"in\": \"\"})\n    def test_addurls_invalid_input(self, path):\n        ds = Dataset(path).create(force=True)\n        in_file = op.join(path, \"in\")\n        for in_type in au.INPUT_TYPES:\n            with assert_raises(IncompleteResultsError) as exc:\n                ds.addurls(in_file, \"{url}\", \"{name}\", input_type=in_type)\n            assert_in(\"Failed to read\", str(exc.exception))\n\n    @with_tree({\"in.csv\": \"url,name,subdir\",\n                \"in.tsv\": \"url\\tname\\tsubdir\",\n                \"in.json\": \"[]\"})\n    def test_addurls_no_rows(self, path):\n        ds = Dataset(path).create(force=True)\n        for fname in [\"in.csv\", \"in.tsv\", \"in.json\"]:\n            with swallow_logs(new_level=logging.WARNING) as cml:\n                assert_in_results(\n                    ds.addurls(fname, \"{url}\", \"{name}\"),\n                    action=\"addurls\",\n                    status=\"notneeded\")\n                cml.assert_logged(\"No rows\", regex=False)\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_stdin_input(self, input_text, input_type, path):\n        ds = Dataset(path).create(force=True)\n        with patch(\"sys.stdin\", new=StringIO(input_text)):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", input_type=input_type)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(ds.path, fname))\n\n    def test_addurls_stdin_input(self):\n        def make_test(text, input_type, description):\n            def fn():\n                self.check_addurls_stdin_input(json_text, \"ext\")\n            fn.description = description\n            return fn\n\n        with open(self.json_file) as jfh:\n            json_text = jfh.read()\n\n        yield make_test(json_text, \"ext\", \"json,default input type\")\n        yield make_test(json_text, \"json\", \"json,json input type\")\n\n        def make_delim_text(delim):\n            row = \"{name}\" + delim + \"{url}\"\n            return \"\\n\".join(\n                [row.format(name=\"name\", url=\"url\")] +\n                [row.format(**rec) for rec in json.loads(json_text)])\n\n        yield make_test(make_delim_text(\",\"), \"csv\", \"csv,csv input type\")\n        yield make_test(make_delim_text(\"\\t\"), \"tsv\", \"tsv,tsv input type\")\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_stdin_input_command_line(self, path):\n        # The previous test checks all the cases, but it overrides sys.stdin.\n        # Do a simple check that's closer to a command line call.\n        Dataset(path).create(force=True)\n        runner = WitlessRunner(cwd=path)\n        with open(self.json_file) as jfh:\n            runner.run([\"datalad\", \"addurls\", '-', '{url}', '{name}'],\n                       stdin=jfh)\n        for fname in [\"a\", \"b\", \"c\"]:\n            ok_exists(op.join(path, fname))\n\n    @with_tempfile(mkdir=True)\n    def test_drop_after(self, path):\n        ds = Dataset(path).create(force=True)\n        ds.repo.set_gitattributes([('a*', {'annex.largefiles': 'nothing'})])\n        # make some files go to git, so we could test that we do not blow\n        # while trying to drop what is in git not annex\n        res = ds.addurls(self.json_file, '{url}', '{name}', drop_after=True)\n\n        assert_result_count(res, 3, action='addurl', status='ok')  # a, b, c  even if a goes to git\n        assert_result_count(res, 2, action='drop', status='ok')  # b, c\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_from_key_invalid_format(self, path):\n        ds = Dataset(path).create(force=True)\n        for fmt in [\"{name}-which-has-no-double-dash\",\n                    # Invalid hash length.\n                    \"MD5-s{size}--{md5sum}a\",\n                    # Invalid hash content.\n                    \"MD5-s{size}--\" + 32 * \"q\"]:\n            with assert_raises(IncompleteResultsError):\n                ds.addurls(self.json_file, \"{url}\", \"{name}\",\n                           key=fmt, exclude_autometa=\"*\")\n\n    @with_tempfile(mkdir=True)\n    def check_addurls_from_key(self, key_arg, expected_backend, fake_dates,\n                               path):\n        ds = Dataset(path).create(force=True, fake_dates=fake_dates)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        ds.addurls(self.json_file, \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                   key=key_arg)\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], expected_backend)\n            assert_false(pstat[\"has_content\"])\n\n        get_res = ds.get(paths, result_renderer=None, on_failure=\"ignore\")\n        assert_result_count(get_res, 2, action=\"get\", status=\"ok\")\n\n    def test_addurls_from_key(self):\n        fn = self.check_addurls_from_key\n        for case in [\n                (fn, \"MD5-s{size}--{md5sum}\", \"MD5\"),\n                (fn, \"MD5E-s{size}--{md5sum}.dat\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5-s{size}--{md5sum}\", \"MD5E\"),\n                (skip_key_tests(fn), \"et:MD5E-s{size}--{md5sum}.dat\", \"MD5\")]:\n            yield case + (False,)\n            yield case + (True,)\n\n    @with_tempfile(mkdir=True)\n    def test_addurls_row_missing_key_fields(self, path):\n        ds = Dataset(path).create(force=True)\n        if OLD_EXAMINEKEY and ds.repo.is_managed_branch():\n            raise SkipTest(\"Adjusted branch functionality requires \"\n                           \"more recent `git annex examinekey`\")\n        data = self.data.copy()\n        for row in data:\n            if row[\"name\"] == \"b\":\n                del row[\"md5sum\"]\n                break\n        with patch(\"sys.stdin\", new=StringIO(json.dumps(data))):\n            ds.addurls(\"-\", \"{url}\", \"{name}\", exclude_autometa=\"*\",\n                       key=\"MD5-s{size}--{md5sum}\")\n\n        repo = ds.repo\n        repo_path = ds.repo.pathobj\n        paths = [repo_path / x for x in \"ac\"]\n\n        annexinfo = repo.get_content_annexinfo(eval_availability=True)\n        for path in paths:\n            pstat = annexinfo[path]\n            eq_(pstat[\"backend\"], \"MD5\")\n            assert_false(pstat[\"has_content\"])\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.13.7.dev656\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 60ms\nInstalled 25 packages in 4.15s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 63 packages in 4.33s\nPrepared 1 package in 3.94s\nInstalled 57 packages in 710ms\n + annexremote==1.6.6\n + appdirs==1.4.4\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.13.7 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2f8ec83b44108d3b022c778a99d648105028f202)\n + deprecated==1.2.15\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.15.0\n + pillow==11.0.0\n + propcache==0.2.1\n + pycparser==2.22\n + pygithub==2.5.0\n + pyjwt==2.10.1\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + soupsieve==2.6\n + tqdm==4.67.1\n + urllib3==1.26.20\n + vcrpy==7.0.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 97 packages in 215ms\nInstalled 32 packages in 866ms\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + filelock==3.16.1\n + gprof2dot==2024.6.6\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + nose-timer==1.0.1\n + platformdirs==4.3.6\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2f8ec83b44108d3b022c778a99d648105028f202\nconfigfile: pyproject.toml\nplugins: cov-6.0.0\ncollected 45 items\n\nr2e_tests/test_1.py ...................x.FFFFFFFFFFFFFFFFFFxFFFxF        [100%]\n\n=================================== FAILURES ===================================\n__________________________ test_addurls_nonannex_repo __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex408tuer1'}\nfilename = '/tmp/datalad_temp_test_addurls_nonannex_repo8r120z1b'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_nonannex_repo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________________ test_addurls_unknown_placeholder _______________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex4vok5e1m'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_addurls_unknown_placeholder'}\nd = '/tmp/datalad_temp_tree_test_addurls_unknown_placeholderh4dq0z0m'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_addurls_unknown_placeholder() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:578: TypeError\n_____________________________ test_addurls_dry_run _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexprvyjfq0'}\nfilename = '/tmp/datalad_temp_test_addurls_dry_runu20ami8_'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dry_run() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n___________________________ TestAddurls.test_addurls ___________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe826d60>,)\nkw = {'path': '/tmp/tmp-page2annexywtsiyt0'}\nfilename = '/tmp/datalad_temp_test_addurls_soir7rs'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n___________________ TestAddurls.test_addurls_unbound_dataset ___________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe826820>,)\nkw = {'path': '/tmp/tmp-page2annex74lnbxuc'}\nfilename = '/tmp/datalad_temp_test_addurls_unbound_datasetg23v9awv'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_unbound_dataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n__________________ TestAddurls.test_addurls_create_newdataset __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe826580>,)\nkw = {'path': '/tmp/tmp-page2annex2b0fz1ky'}\nfilename = '/tmp/datalad_temp_test_addurls_create_newdataset8tj_ih39'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_create_newdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_from_list ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe64c640>,)\nkw = {'path': '/tmp/tmp-page2annexn3l_8rvy'}\nfilename = '/tmp/datalad_temp_test_addurls_from_listoxf04n3a'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_from_list() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_____________________ TestAddurls.test_addurls_subdataset ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe64c400>,)\nkw = {'path': '/tmp/tmp-page2annex97ceppsd'}\nfilename = '/tmp/datalad_temp_test_addurls_subdataset3qrfjr_k'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_subdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_repindex _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe7e1130>,)\nkw = {'path': '/tmp/tmp-page2annexpsvi98ie'}\nfilename = '/tmp/datalad_temp_test_addurls_repindexagcqhokp'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_repindex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_url_parts ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbeabc3a0>,)\nkw = {'path': '/tmp/tmp-page2annexlzaabhvk'}\nfilename = '/tmp/datalad_temp_test_addurls_url_partsx0a0taam'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_parts() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n____________________ TestAddurls.test_addurls_url_filename _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe8265b0>,)\nkw = {'path': '/tmp/tmp-page2annexr4fhaika'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filenamee0t2la4e'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n__________________ TestAddurls.test_addurls_url_filename_fail __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe826340>,)\nkw = {'path': '/tmp/tmp-page2annexi08eb4x9'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filename_failgfx8j4hr'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename_fail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_metafail _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe826970>,)\nkw = {'path': '/tmp/tmp-page2annex192oj54r'}\nfilename = '/tmp/datalad_temp_test_addurls_metafailgdnnq_9r'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_metafail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n____________________ TestAddurls.test_addurls_dropped_urls _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4f88e0>,)\nkw = {'path': '/tmp/tmp-page2annex8fi6o4_m'}\nfilename = '/tmp/datalad_temp_test_addurls_dropped_urlsl5ruxi0e'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dropped_urls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________________ TestAddurls.test_addurls_version _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4f8700>,)\nkw = {'path': '/tmp/tmp-page2annexla373daw'}\nfilename = '/tmp/datalad_temp_test_addurls_versions2us4tr2'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_version() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________________ TestAddurls.test_addurls_deeper ________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4f8520>,)\nkw = {'path': '/tmp/tmp-page2annexakb_f_d2'}\nfilename = '/tmp/datalad_temp_test_addurls_deeper6oku7i4e'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_deeper() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n____________________ TestAddurls.test_addurls_invalid_input ____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4f8130>,)\nkw = {'path': '/tmp/tmp-page2annexow4lp3ek'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_addurls_invalid_input'}\nd = '/tmp/datalad_temp_tree_test_addurls_invalid_inputvax374ad'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_addurls_invalid_input() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:578: TypeError\n_______________________ TestAddurls.test_addurls_no_rows _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4f8310>,)\nkw = {'path': '/tmp/tmp-page2annex78dpsqg_'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_addurls_no_rows'}\nd = '/tmp/datalad_temp_tree_test_addurls_no_rowssqb0orch'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_addurls_no_rows() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:578: TypeError\n______________ TestAddurls.test_addurls_stdin_input_command_line _______________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4f8ee0>,)\nkw = {'path': '/tmp/tmp-page2annex5pjo995t'}\nfilename = '/tmp/datalad_temp_test_addurls_stdin_input_command_linexp89x0gx'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_stdin_input_command_line() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_________________________ TestAddurls.test_drop_after __________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4f8d00>,)\nkw = {'path': '/tmp/tmp-page2annexuitrwu8y'}\nfilename = '/tmp/datalad_temp_test_drop_afterzznmhi24'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_drop_after() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________ TestAddurls.test_addurls_from_key_invalid_format _______________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4cebb0>,)\nkw = {'path': '/tmp/tmp-page2annex_tcwmv8d'}\nfilename = '/tmp/datalad_temp_test_addurls_from_key_invalid_format0oj0xuls'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_from_key_invalid_format() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________ TestAddurls.test_addurls_row_missing_key_fields ________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x75ecbe4ce610>,)\nkw = {'path': '/tmp/tmp-page2annexz5jv759c'}\nfilename = '/tmp/datalad_temp_test_addurls_row_missing_key_fieldsj6bhd2ag'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_row_missing_key_fields() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_lower_case\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_formatter_missing_arg\nPASSED r2e_tests/test_1.py::test_repformatter\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_get_subpaths\nPASSED r2e_tests/test_1.py::test_sort_paths\nPASSED r2e_tests/test_1.py::test_is_legal_metafield\nPASSED r2e_tests/test_1.py::test_filter_legal_metafield\nPASSED r2e_tests/test_1.py::test_fmt_to_name\nPASSED r2e_tests/test_1.py::test_get_file_parts\nPASSED r2e_tests/test_1.py::test_get_url_parts\nPASSED r2e_tests/test_1.py::test_extract\nPASSED r2e_tests/test_1.py::test_extract_disable_autometa\nPASSED r2e_tests/test_1.py::test_extract_exclude_autometa_regexp\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\nXFAIL r2e_tests/test_1.py::test_extract_csv_tsv_json_equal - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_extract_csv_tsv_json_equal will be ignored\nXFAIL r2e_tests/test_1.py::TestAddurls::test_addurls_stdin_input - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_addurls_stdin_input will be ignored\nXFAIL r2e_tests/test_1.py::TestAddurls::test_addurls_from_key - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_addurls_from_key will be ignored\nFAILED r2e_tests/test_1.py::test_addurls_nonannex_repo - TypeError: test_addu...\nFAILED r2e_tests/test_1.py::test_addurls_unknown_placeholder - TypeError: tes...\nFAILED r2e_tests/test_1.py::test_addurls_dry_run - TypeError: test_addurls_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls - TypeError: test_addur...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_unbound_dataset - TypeE...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_create_newdataset - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_from_list - TypeError: ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_subdataset - TypeError:...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_repindex - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_parts - TypeError: ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename_fail - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_metafail - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_dropped_urls - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_version - TypeError: te...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_deeper - TypeError: tes...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_invalid_input - TypeErr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_no_rows - TypeError: te...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_stdin_input_command_line\nFAILED r2e_tests/test_1.py::TestAddurls::test_drop_after - TypeError: test_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_from_key_invalid_format\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_row_missing_key_fields\n=================== 22 failed, 20 passed, 3 xfailed in 4.34s ===================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2f8ec83b44108d3b022c778a99d648105028f202\nconfigfile: pyproject.toml\nplugins: cov-6.0.0\ncollected 45 items\n\nr2e_tests/test_1.py ................FFFx.FFFFFFFFFFFFFFFFFFxFFFxF        [100%]\n\n=================================== FAILURES ===================================\n_________________________________ test_extract _________________________________\n\n    @known_failure_githubci_win\n    def test_extract():\n>       info, subpaths = au.extract(\n            ST_DATA[\"rows\"],\n            url_format=\"{name}_{debut_season}.com\",\n            filename_format=\"{age_group}//{now_dead}//{name}.csv\")\nE       TypeError: extract() missing 1 required positional argument: 'input_type'\n\nr2e_tests/test_1.py:258: TypeError\n________________________ test_extract_disable_autometa _________________________\n\n    def test_extract_disable_autometa():\n>       info, _ = au.extract(\n            ST_DATA[\"rows\"],\n            url_format=\"{name}_{debut_season}.com\",\n            filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n            exclude_autometa=\"*\",\n            meta=[\"group={age_group}\"])\nE       TypeError: extract() missing 1 required positional argument: 'input_type'\n\nr2e_tests/test_1.py:289: TypeError\n_____________________ test_extract_exclude_autometa_regexp _____________________\n\n    def test_extract_exclude_autometa_regexp():\n>       info, _ = au.extract(\n            ST_DATA[\"rows\"],\n            url_format=\"{name}_{debut_season}.com\",\n            filename_format=\"{age_group}//{now_dead}//{name}.csv\",\n            exclude_autometa=\"ea\")\nE       TypeError: extract() missing 1 required positional argument: 'input_type'\n\nr2e_tests/test_1.py:302: TypeError\n__________________________ test_addurls_nonannex_repo __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexclzzhcw7'}\nfilename = '/tmp/datalad_temp_test_addurls_nonannex_repopw3a9h8q'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_nonannex_repo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________________ test_addurls_unknown_placeholder _______________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexorjjxvz_'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_addurls_unknown_placeholder'}\nd = '/tmp/datalad_temp_tree_test_addurls_unknown_placeholder6yk2duis'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_addurls_unknown_placeholder() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:578: TypeError\n_____________________________ test_addurls_dry_run _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexn6numb5c'}\nfilename = '/tmp/datalad_temp_test_addurls_dry_run5ty78qyt'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dry_run() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n___________________________ TestAddurls.test_addurls ___________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767765f5f10>,)\nkw = {'path': '/tmp/tmp-page2annexk9mcozz7'}\nfilename = '/tmp/datalad_temp_test_addurlsqfjxuz9g'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n___________________ TestAddurls.test_addurls_unbound_dataset ___________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767765f5e20>,)\nkw = {'path': '/tmp/tmp-page2annex_rhjae77'}\nfilename = '/tmp/datalad_temp_test_addurls_unbound_datasetl71bd6wp'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_unbound_dataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n__________________ TestAddurls.test_addurls_create_newdataset __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767765f5580>,)\nkw = {'path': '/tmp/tmp-page2annex0ziwdr7l'}\nfilename = '/tmp/datalad_temp_test_addurls_create_newdataset374by5ju'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_create_newdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_from_list ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767765e9eb0>,)\nkw = {'path': '/tmp/tmp-page2annexc84083kh'}\nfilename = '/tmp/datalad_temp_test_addurls_from_listubb1gse1'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_from_list() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_____________________ TestAddurls.test_addurls_subdataset ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767765e9a00>,)\nkw = {'path': '/tmp/tmp-page2annexoqz68q4f'}\nfilename = '/tmp/datalad_temp_test_addurls_subdatasetuindybxj'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_subdataset() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_repindex _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767767b3580>,)\nkw = {'path': '/tmp/tmp-page2annexmmwikuk5'}\nfilename = '/tmp/datalad_temp_test_addurls_repindexpnfa7i_h'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_repindex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_url_parts ______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x776776604100>,)\nkw = {'path': '/tmp/tmp-page2annexta7f64nu'}\nfilename = '/tmp/datalad_temp_test_addurls_url_parts2z1ea1gt'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_parts() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n____________________ TestAddurls.test_addurls_url_filename _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767766042e0>,)\nkw = {'path': '/tmp/tmp-page2annexkrx0zw8g'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filename9w0inn7d'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n__________________ TestAddurls.test_addurls_url_filename_fail __________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x7767766044f0>,)\nkw = {'path': '/tmp/tmp-page2annex2h55zlp0'}\nfilename = '/tmp/datalad_temp_test_addurls_url_filename_failsh13dhpw'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_url_filename_fail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n______________________ TestAddurls.test_addurls_metafail _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x776776604700>,)\nkw = {'path': '/tmp/tmp-page2annexh2bw58m8'}\nfilename = '/tmp/datalad_temp_test_addurls_metafailbj9pmpyn'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_metafail() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n____________________ TestAddurls.test_addurls_dropped_urls _____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x776776604910>,)\nkw = {'path': '/tmp/tmp-page2annexo591urg6'}\nfilename = '/tmp/datalad_temp_test_addurls_dropped_urlshw4sjqq7'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_dropped_urls() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________________ TestAddurls.test_addurls_version _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x776776604b20>,)\nkw = {'path': '/tmp/tmp-page2annexah5n18gc'}\nfilename = '/tmp/datalad_temp_test_addurls_version0xi50nql'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_version() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________________ TestAddurls.test_addurls_deeper ________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x776776604d30>,)\nkw = {'path': '/tmp/tmp-page2annex0c4reft8'}\nfilename = '/tmp/datalad_temp_test_addurls_deeperf3gjfpdv'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_deeper() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n____________________ TestAddurls.test_addurls_invalid_input ____________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x776776604f40>,)\nkw = {'path': '/tmp/tmp-page2annexbo_9wlqg'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_addurls_invalid_input'}\nd = '/tmp/datalad_temp_tree_test_addurls_invalid_inputkdfak_s2'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_addurls_invalid_input() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:578: TypeError\n_______________________ TestAddurls.test_addurls_no_rows _______________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x77677660a190>,)\nkw = {'path': '/tmp/tmp-page2annexqm3fbat0'}\ntkwargs_ = {'dir': None, 'prefix': 'datalad_temp_tree_test_addurls_no_rows'}\nd = '/tmp/datalad_temp_tree_test_addurls_no_rowsci20z2sl'\n\n    @wraps(t)\n    def  _wrap_with_tree(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_addurls_no_rows() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:578: TypeError\n______________ TestAddurls.test_addurls_stdin_input_command_line _______________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x77677660a5b0>,)\nkw = {'path': '/tmp/tmp-page2annexsk05cbf0'}\nfilename = '/tmp/datalad_temp_test_addurls_stdin_input_command_lineodq7cra9'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_stdin_input_command_line() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_________________________ TestAddurls.test_drop_after __________________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x77677660a790>,)\nkw = {'path': '/tmp/tmp-page2annexpc_sahq4'}\nfilename = '/tmp/datalad_temp_test_drop_after9g4ncjuf'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_drop_after() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________ TestAddurls.test_addurls_from_key_invalid_format _______________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x77677660a9a0>,)\nkw = {'path': '/tmp/tmp-page2annexrd1dfhtg'}\nfilename = '/tmp/datalad_temp_test_addurls_from_key_invalid_formati349muxd'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_from_key_invalid_format() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n_______________ TestAddurls.test_addurls_row_missing_key_fields ________________\n\narg = (<r2e_tests.test_1.TestAddurls object at 0x77677660adc0>,)\nkw = {'path': '/tmp/tmp-page2annexfyo7s_c2'}\nfilename = '/tmp/datalad_temp_test_addurls_row_missing_key_fieldskhuh4i1r'\n\n    @wraps(t)\n    def  _wrap_with_tempfile(*arg, **kw):\n        if 'dir' not in tkwargs.keys():\n            # if not specified otherwise, respect datalad.tests.temp.dir config\n            # as this is a test helper\n            tkwargs['dir'] = dl_cfg.get(\"datalad.tests.temp.dir\")\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_addurls_row_missing_key_fields() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:757: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_formatter\nPASSED r2e_tests/test_1.py::test_formatter_lower_case\nPASSED r2e_tests/test_1.py::test_formatter_no_idx_map\nPASSED r2e_tests/test_1.py::test_formatter_no_mapping_arg\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_with_spaces\nPASSED r2e_tests/test_1.py::test_formatter_placeholder_nonpermitted_chars\nPASSED r2e_tests/test_1.py::test_formatter_missing_arg\nPASSED r2e_tests/test_1.py::test_repformatter\nPASSED r2e_tests/test_1.py::test_clean_meta_args\nPASSED r2e_tests/test_1.py::test_get_subpaths\nPASSED r2e_tests/test_1.py::test_sort_paths\nPASSED r2e_tests/test_1.py::test_is_legal_metafield\nPASSED r2e_tests/test_1.py::test_filter_legal_metafield\nPASSED r2e_tests/test_1.py::test_fmt_to_name\nPASSED r2e_tests/test_1.py::test_get_file_parts\nPASSED r2e_tests/test_1.py::test_get_url_parts\nPASSED r2e_tests/test_1.py::test_extract_wrong_input_type\nXFAIL r2e_tests/test_1.py::test_extract_csv_tsv_json_equal - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_extract_csv_tsv_json_equal will be ignored\nXFAIL r2e_tests/test_1.py::TestAddurls::test_addurls_stdin_input - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_addurls_stdin_input will be ignored\nXFAIL r2e_tests/test_1.py::TestAddurls::test_addurls_from_key - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_addurls_from_key will be ignored\nFAILED r2e_tests/test_1.py::test_extract - TypeError: extract() missing 1 req...\nFAILED r2e_tests/test_1.py::test_extract_disable_autometa - TypeError: extrac...\nFAILED r2e_tests/test_1.py::test_extract_exclude_autometa_regexp - TypeError:...\nFAILED r2e_tests/test_1.py::test_addurls_nonannex_repo - TypeError: test_addu...\nFAILED r2e_tests/test_1.py::test_addurls_unknown_placeholder - TypeError: tes...\nFAILED r2e_tests/test_1.py::test_addurls_dry_run - TypeError: test_addurls_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls - TypeError: test_addur...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_unbound_dataset - TypeE...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_create_newdataset - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_from_list - TypeError: ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_subdataset - TypeError:...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_repindex - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_parts - TypeError: ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_url_filename_fail - Typ...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_metafail - TypeError: t...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_dropped_urls - TypeErro...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_version - TypeError: te...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_deeper - TypeError: tes...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_invalid_input - TypeErr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_no_rows - TypeError: te...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_stdin_input_command_line\nFAILED r2e_tests/test_1.py::TestAddurls::test_drop_after - TypeError: test_dr...\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_from_key_invalid_format\nFAILED r2e_tests/test_1.py::TestAddurls::test_addurls_row_missing_key_fields\n=================== 25 failed, 17 passed, 3 xfailed in 2.00s ===================\n",
    "old_commit_res_stderr": ""
}
datalad/plugin/addurls.py datalad/plugin/tests/test_addurls.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Extract Function Fails in Tests Due to Missing `input_type` Parameter

**Description:**
After recent changes to the `extract` function to support passing a list of records, several tests are failing because they still call `extract` without the required `input_type` argument.

**Example Buggy Code:**
```python
info, subpaths = au.extract(
    ST_DATA['rows'],
    url_format='{name}_{debut_season}.com',
    filename_format='{age_group}//{now_dead}//{name}.csv'
)
```

**Expected Behavior:**
The `extract` function should process the provided records correctly without requiring the `input_type` parameter, accommodating the new functionality of accepting a list of records.

**Actual Behavior:**
Calling `extract` without the `input_type` parameter results in the following error:
```
TypeError: extract() missing 1 required positional argument: 'input_type'
```
This prevents the tests from executing successfully and indicates a mismatch between the function's updated signature and its usage in the tests.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections.abc import Mapping

import logging
import os
import re
import string
import sys

from functools import partial
from urllib.parse import urlparse

from datalad.support.external_versions import external_versions
import datalad.support.path as op
from datalad.distribution.dataset import resolve_path
from datalad.dochelpers import exc_str
from datalad.log import log_progress, with_result_progress
from datalad.interface.base import Interface
from datalad.interface.base import build_doc
from datalad.interface.results import annexjson2result, get_status_dict
from datalad.interface.common_opts import (
    jobs_opt,
    nosave_opt,
)
from datalad.support.exceptions import CommandError
from datalad.support.itertools import groupby_sorted
from datalad.support.network import get_url_filename
from datalad.support.path import split_ext
from datalad.support.parallel import (
    ProducerConsumerProgressLog,
    no_parentds_in_futures,
)
from datalad.support.s3 import get_versioned_url
from datalad.utils import (
    ensure_list,
    get_suggestions_msg,
    Path,
    unlink,
)

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    missing : str, optional
        When column lookup results in an empty string, use this value in
        its place.
    """

    def __init__(self, idx_to_name=None, missing_value=None):
        self.idx_to_name = idx_to_name or {}
        self.missing = missing_value

    def format(self, format_string, *args, **kwargs):
        if not isinstance(args[0], Mapping):
            raise ValueError("First positional argument should be mapping")
        return super(Formatter, self).format(format_string, *args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]

        name = key
        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            name = self.idx_to_name[key_int]

        try:
            value = data[name]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)

        if self.missing is not None and isinstance(value, str):
            return value or self.missing
        return value

    def convert_field(self, value, conversion):
        if conversion == 'l':
            return str(value).lower()
        return super(Formatter, self).convert_field(value, conversion)


class RepFormatter(Formatter):
    """Extend Formatter to support a {_repindex} placeholder.
    """

    def __init__(self, *args, **kwargs):
        super(RepFormatter, self).__init__(*args, **kwargs)
        self.repeats = {}
        self.repindex = 0

    def format(self, *args, **kwargs):
        self.repindex = 0
        result = super(RepFormatter, self).format(*args, **kwargs)
        if result in self.repeats:
            self.repindex = self.repeats[result] + 1
            self.repeats[result] = self.repindex
            result = super(RepFormatter, self).format(*args, **kwargs)
        else:
            self.repeats[result] = 0
        return result

    def get_value(self, key, args, kwargs):
        args[0]["_repindex"] = self.repindex
        return super(RepFormatter, self).get_value(key, args, kwargs)


def clean_meta_args(args):
    """Process metadata arguments.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    A dict mapping field names to values.
    """
    results = {}
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            raise ValueError("meta argument isn't in 'field=value' format")

        if not value:
            # The `url_file` may have an empty value.
            continue
        results[field] = value
    return results


def get_subpaths(filename):
    """Convert "//" marker in `filename` to a list of subpaths.

    >>> from datalad.plugin.addurls import get_subpaths
    >>> get_subpaths("p1/p2//p3/p4//file")
    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])

    Note: With Python 3, the subpaths could be generated with

        itertools.accumulate(filename.split("//")[:-1], os.path.join)

    Parameters
    ----------
    filename : str
        File name with "//" marking subpaths.

    Returns
    -------
    A tuple of the filename with any "//" collapsed to a single
    separator and a list of subpaths (str).
    """
    if "//" not in filename:
        return filename, []

    spaths = []
    for part in filename.split("//")[:-1]:
        path = os.path.join(spaths[-1], part) if spaths else part
        spaths.append(path)
    return filename.replace("//", os.path.sep), spaths


def is_legal_metafield(name):
    """Test whether `name` is a valid metadata field.

    The set of permitted characters is taken from git-annex's
    MetaData.hs:legalField.
    """
    return bool(re.match(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z", name))


def filter_legal_metafield(fields):
    """Remove illegal names from `fields`.

    Note: This is like `filter(is_legal_metafield, fields)` but the
    dropped values are logged.
    """
    legal = []
    for field in fields:
        if is_legal_metafield(field):
            legal.append(field)
        else:
            lgr.debug("%s is not a valid metadata field name; dropping",
                      field)
    return legal


class AnnexKeyParser(object):
    """Parse a full annex key into subparts.

    The key may have an "et:" prefix appended, which signals that the backend's
    extension state should be toggled.

    See <https://git-annex.branchable.com/internals/key_format/>.

    Parameters
    ----------
    format_fn : callable
        Function that takes a format string and a row and returns the full key.
    format_string : str
        Format string for the full key.
    """

    def __init__(self, format_fn, format_string):
        self.format_fn = format_fn
        self.format_string = format_string
        self.regexp = re.compile(r"(?P<et>et:)?"
                                 r"(?P<backend>[A-Z0-9]+)"
                                 r"(?:-[^-]+)?"
                                 r"--(?P<keyname>[^/\n]+)\Z")
        self.empty = "".join(i[0]
                             for i in string.Formatter().parse(format_string))

    @staticmethod
    def _validate_common(backend, key):
        if backend.endswith("E"):
            ext = True
            backend = backend[:-1]
        else:
            ext = False

        expected_lengths = {"MD5": 32,
                            "SHA1": 40,
                            "SHA256": 64}

        expected_len = expected_lengths.get(backend)
        if not expected_len:
            return

        if ext and "." in key:
            key = key.split(".", maxsplit=1)[0]

        if len(key) != expected_len:
            raise ValueError("{} key does not have expected length of {}: {}"
                             .format(backend, expected_len, key))

        if not re.match(r"[a-f0-9]+\Z", key):
            raise ValueError("{} key has non-hexadecimal character: {}"
                             .format(backend, key))

    def parse(self, row):
        """Format the key with the fields in `row` and parse it.

        Returns
        -------
        A dictionary with the following keys that match their counterparts in
        the output of `git annex examinekey --json`: "key" (the full annex
        key), "backend", and "keyname". If the key had an "et:" prefix, there
        is also a "target_backend" key.

        Raises
        ------
        ValueError if the formatted value doesn't look like a valid key
        """
        try:
            key = self.format_fn(self.format_string, row)
        except KeyError as exc:
            lgr.debug("Row missing fields for --key: %s",
                      exc_str(exc))
            return {}

        if key == self.empty:
            lgr.debug("All fields in --key's value are empty in row: %s", row)
            # We got the same string that'd we get if all the fields were
            # empty, so this doesn't have a key.
            return {}

        match = self.regexp.match(key)
        if match:
            info = match.groupdict()
            et = info.pop("et", None)
            if et:
                info["key"] = key[3:]  # Drop "et:" from full key.
                backend = info["backend"]
                if backend.endswith("E"):
                    info["target_backend"] = backend[:-1]
                else:
                    info["target_backend"] = backend + "E"
            else:
                info["key"] = key
            self._validate_common(info["backend"], info["keyname"])
            return info
        else:
            raise ValueError(
                "Key does not match expected "
                "[et:]<backend>[-[CmsS]NNN]--<key> format: {}"
                .format(key))


def get_fmt_names(format_string):
    """Yield field names in `format_string`.
    """
    for _, name, _, _ in string.Formatter().parse(format_string):
        if name:
            yield name


def fmt_to_name(format_string, num_to_name):
    """Try to map a format string to a single name.

    Parameters
    ----------
    format_string : string
    num_to_name : dict
        A dictionary that maps from an integer to a column name.  This
        enables mapping the format string to an integer to a name.

    Returns
    -------
    A placeholder name if `format_string` consists of a single
    placeholder and no other text.  Otherwise, None is returned.
    """
    parsed = list(string.Formatter().parse(format_string))
    if len(parsed) != 1:
        # It's an empty string or there's more than one placeholder.
        return
    if parsed[0][0]:
        # Format string contains text before the placeholder.
        return

    name = parsed[0][1]
    if not name:
        # The field name is empty.
        return

    try:
        return num_to_name[int(name)]
    except (KeyError, ValueError):
        return name


INPUT_TYPES = ["ext", "csv", "tsv", "json"]


def _read(stream, input_type):
    if input_type in ["csv", "tsv"]:
        import csv
        csvrows = csv.reader(stream,
                             delimiter="\t" if input_type == "tsv" else ",")
        try:
            headers = next(csvrows)
        except StopIteration:
            raise ValueError("Failed to read {} rows from {}"
                             .format(input_type.upper(), stream))
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        idx_map = dict(enumerate(headers))
        rows = [dict(zip(headers, r)) for r in csvrows]
    elif input_type == "json":
        import json
        try:
            rows = json.load(stream)
        except json.decoder.JSONDecodeError as e:
            raise ValueError(
                "Failed to read JSON from stream {}: {}"
                .format(stream, exc_str(e)))
        # For json input, we do not support indexing by position,
        # only names.
        idx_map = {}
    else:
        raise ValueError(
            "input_type {} is invalid. Known values: {}"
            .format(input_type, ", ".join(INPUT_TYPES)))
    return rows, idx_map


def _get_placeholder_exception(exc, msg_prefix, known):
    """Recast KeyError as a ValueError with close-match suggestions.
    """
    value = exc.args[0]
    if isinstance(value, str):
        sugmsg = get_suggestions_msg(value, known)
    else:
        sugmsg = "Out-of-bounds or unsupported index."
    # Note: Keeping this a KeyError is probably more appropriate but then the
    # entire message, which KeyError takes as the key, will be rendered with
    # outer quotes.
    return ValueError("{}: {}{}{}"
                      .format(msg_prefix, exc, ". " if sugmsg else "", sugmsg))


def _format_filenames(format_fn, rows, row_infos):
    subpaths = set()
    for row, info in zip(rows, row_infos):
        try:
            filename = format_fn(row)
        except KeyError as exc:
            raise _get_placeholder_exception(
                exc, "Unknown placeholder in file name", row)
        filename, spaths = get_subpaths(filename)
        subpaths |= set(spaths)
        info["filename"] = filename
        info["subpath"] = spaths[-1] if spaths else None
    return subpaths


def get_file_parts(filename, prefix="name"):
    """Assign a name to various parts of a file.

    Parameters
    ----------
    filename : str
        A file name (no leading path is permitted).
    prefix : str
        Prefix to prepend to the key names.

    Returns
    -------
    A dict mapping each part to a value.
    """
    root, ext = split_ext(filename)
    root_py, ext_py = os.path.splitext(filename)

    return {prefix: filename,
            prefix + "_root": root,
            prefix + "_ext": ext,
            prefix + "_root_py": root_py,
            prefix + "_ext_py": ext_py}


def get_url_parts(url):
    """Assign a name to various parts of the URL.

    Parameters
    ----------
    url : str

    Returns
    -------
    A dict with keys `_url_hostname` and, for a path with N+1 parts,
    '_url0' through '_urlN' .  There is also a `_url_basename` key for
    the rightmost part of the path.
    """
    parsed = urlparse(url)
    if not parsed.netloc:
        return {}

    names = {"_url_hostname": parsed.netloc}

    path = parsed.path.strip("/")
    if not path:
        return names

    url_parts = path.split("/")
    for pidx, part in enumerate(url_parts):
        names["_url{}".format(pidx)] = part
    basename = url_parts[-1]
    names["_url_basename"] = basename
    names.update(get_file_parts(basename, prefix="_url_basename"))
    return names


def add_extra_filename_values(filename_format, rows, urls, dry_run):
    """Extend `rows` with values for special formatting fields.
    """
    file_fields = list(get_fmt_names(filename_format))
    if any(i.startswith("_url") for i in file_fields):
        for row, url in zip(rows, urls):
            row.update(get_url_parts(url))

    if any(i.startswith("_url_filename") for i in file_fields):
        if dry_run:  # Don't waste time making requests.
            dummy = get_file_parts("BASE.EXT", "_url_filename")
            for idx, row in enumerate(rows):
                row.update(
                    {k: v + str(idx) for k, v in dummy.items()})
        else:
            num_urls = len(urls)
            log_progress(lgr.info, "addurls_requestnames",
                         "Requesting file names for %d URLs", num_urls,
                         label="Requesting names", total=num_urls,
                         unit=" Files")
            for row, url in zip(rows, urls):
                # If we run into any issues here, we're just going to raise an
                # exception and then abort inside dlplugin.  It'd be good to
                # disentangle this from `extract` so that we could yield an
                # individual error, drop the row, and keep going.
                filename = get_url_filename(url)
                if filename:
                    row.update(get_file_parts(filename, "_url_filename"))
                else:
                    raise ValueError(
                        "{} does not contain a filename".format(url))
                log_progress(lgr.info, "addurls_requestnames",
                             "%s returned for %s", url, filename,
                             update=1, increment=True)
            log_progress(lgr.info, "addurls_requestnames",
                         "Finished requesting file names")


def sort_paths(paths):
    """Sort `paths` by directory level and then alphabetically.

    Parameters
    ----------
    paths : iterable of str

    Returns
    -------
    Generator of sorted paths.
    """
    def level_and_name(p):
        return p.count(os.path.sep), p

    yield from sorted(paths, key=level_and_name)


def extract(stream, input_type, url_format="{0}", filename_format="{1}",
            exclude_autometa=None, meta=None, key=None,
            dry_run=False, missing_value=None):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.
    input_type : {'csv', 'tsv', 'json'}

    All other parameters match those described in `AddUrls`.

    Returns
    -------
    A tuple where the first item is a list with a dict of extracted information
    for each row in `stream` and the second item a list subdataset paths,
    sorted breadth-first.
    """
    meta = ensure_list(meta)

    rows, colidx_to_name = _read(stream, input_type)
    if not rows:
        lgr.warning("No rows found in %s", stream)
        return [], []

    # Formatter for everything but file names
    fmt = Formatter(colidx_to_name, missing_value)
    format_url = partial(fmt.format, url_format)

    auto_meta_args = []
    if exclude_autometa not in ["*", ""]:
        urlcol = fmt_to_name(url_format, colidx_to_name)
        # TODO: Try to normalize invalid fields, checking for any
        # collisions.
        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)
        if exclude_autometa:
            metacols = (c for c in metacols
                        if not re.search(exclude_autometa, c))
        metacols = filter_legal_metafield(metacols)
        auto_meta_args = [c + "=" + "{" + c + "}" for c in metacols]

    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]

    info_fns = []
    if formats_meta:
        def set_meta_args(info, row):
            info["meta_args"] = clean_meta_args(fmt(row)
                                                for fmt in formats_meta)
        info_fns.append(set_meta_args)
    if key:
        key_parser = AnnexKeyParser(fmt.format, key)

        def set_key(info, row):
            info["key"] = key_parser.parse(row)
        info_fns.append(set_key)

    rows_with_url = []
    infos = []
    for row in rows:
        try:
            url = format_url(row)
        except KeyError as exc:
            raise _get_placeholder_exception(
                exc, "Unknown placeholder in URL", row)
        if not url or url == missing_value:
            continue  # pragma: no cover, peephole optimization
        rows_with_url.append(row)
        info = {"url": url}
        for fn in info_fns:
            fn(info, row)
        infos.append(info)

    n_dropped = len(rows) - len(rows_with_url)
    if n_dropped:
        lgr.warning("Dropped %d row(s) that had an empty URL", n_dropped)

    # Format the filename in a second pass so that we can provide
    # information about the formatted URLs.
    add_extra_filename_values(filename_format, rows_with_url,
                              [i["url"] for i in infos],
                              dry_run)

    # For the file name, we allow the _repindex special key.
    format_filename = partial(
        RepFormatter(colidx_to_name, missing_value).format,
        filename_format)
    subpaths = _format_filenames(format_filename, rows_with_url, infos)
    return infos, list(sort_paths(subpaths))


def _add_url(row, ds, repo, options=None, drop_after=False):
    filename_abs = row["filename_abs"]
    filename = row["ds_filename"]
    try:
        out_json = repo.add_url_to_file(filename, row["url"],
                                        batch=True, options=options)
    except CommandError as exc:
        yield get_status_dict(action="addurls",
                              ds=ds,
                              type="file",
                              path=filename_abs,
                              message=exc_str(exc),
                              status="error")
        return

    # In the case of an error, the json object has file=None.
    if out_json["file"] is None:
        out_json["file"] = filename_abs
    res_addurls = annexjson2result(
        out_json, ds, action="addurls",
        type="file", logger=lgr)
    yield res_addurls

    if not res_addurls["status"] == "ok":
        return

    if drop_after and 'annexkey' in res_addurls:
        # unfortunately .drop has no batched mode, and drop_key ATM would
        # raise AssertionError if not success, and otherwise return nothing
        try:
            repo.drop_key(res_addurls['annexkey'], batch=True)
            st_kwargs = dict(status="ok")
        except (AssertionError, CommandError) as exc:
            st_kwargs = dict(message=exc_str(exc),
                             status="error")
        yield get_status_dict(action="drop",
                              ds=ds,
                              annexkey=res_addurls['annexkey'],
                              type="file",
                              path=filename_abs,
                              **st_kwargs)


class RegisterUrl(object):
    """Create files (without content) from user-supplied keys and register URLs.
    """

    def __init__(self, ds, repo=None):
        self.ds = ds
        self.repo = repo or ds.repo
        self._err_res = get_status_dict(action="addurls", ds=self.ds,
                                        type="file", status="error")
        self.use_pointer = repo.is_managed_branch()

    def examinekey(self, parsed_key, filename, migrate=False):
        opts = []
        if migrate:
            opts.append("--migrate-to-backend=" + parsed_key["target_backend"])
        opts.extend(["--filename=" + filename, parsed_key["key"]])
        return self.repo.call_annex_records(["examinekey"] + opts)[0]

    def fromkey(self, key, filename):
        return self.repo.call_annex_records(
            ["fromkey", "--force", key, filename])[0]

    def registerurl(self, key, url):
        self.repo.call_annex(["registerurl", key, url])

    def _write_pointer(self, row, ek_info):
        try:
            fname = Path(row["filename_abs"])
            fname.parent.mkdir(exist_ok=True, parents=True)
            fname.write_text(ek_info["objectpointer"])
        except Exception as exc:
            message = exc_str(exc)
            status = "error"
        else:
            message = "registered URL"
            status = "ok"
        return get_status_dict(action="addurls", ds=self.ds, type="file",
                               status=status, message=message)

    def __call__(self, row):
        filename = row["ds_filename"]
        try:
            parsed_key = row["key"]
            migrate = "target_backend" in parsed_key
            use_pointer = self.use_pointer
            ek_info = None
            if use_pointer or migrate:
                ek_info = self.examinekey(parsed_key, filename,
                                          migrate=migrate)
                if not ek_info:
                    yield dict(self._err_res,
                               path=row["filename_abs"],
                               message=("Failed to get information for %s",
                                        parsed_key))
                    return
                key = ek_info["key"]
            else:
                key = parsed_key["key"]

            self.registerurl(key, row["url"])
            if use_pointer:
                res = self._write_pointer(row, ek_info)
            else:
                res = annexjson2result(self.fromkey(key, filename),
                                       self.ds, type="file", logger=lgr)
                if not res.get("message"):
                    res["message"] = "registered URL"
        except CommandError as exc:
            yield dict(self._err_res,
                       path=row["filename_abs"],
                       message=exc_str(exc))
        else:
            yield res


# Note: If any other modules end up needing these batch operations, this should
# find a new home.


class BatchedRegisterUrl(RegisterUrl):
    """Like `RegisterUrl`, but use batched commands underneath.
    """

    def __init__(self, ds, repo=None):
        super().__init__(ds, repo)
        self._batch_commands = {}

    def _batch(self, command, batch_input,
               output_proc=None, json=False, batch_options=None):
        bcmd = self._batch_commands.get(command)
        if not bcmd:
            repo = self.repo
            bcmd = repo._batched.get(
                command, path=repo.path, json=json, output_proc=output_proc,
                annex_options=batch_options)
            self._batch_commands[command] = bcmd
        return bcmd(batch_input)

    def examinekey(self, parsed_key, filename, migrate=False):
        if migrate:
            opts = ["--migrate-to-backend=" + parsed_key["target_backend"]]
        else:
            opts = None
        return self._batch("examinekey", (parsed_key["key"], filename),
                           json=True, batch_options=opts)

    def fromkey(self, key, filename):
        return self._batch("fromkey", (key, filename), json=True,
                           # --force is needed because the key (usually) does
                           # not exist in the local repository.
                           batch_options=["--force"])

    @staticmethod
    def _ignore(_stdout):
        return

    def registerurl(self, key, url):
        self._batch("registerurl", (key, url),
                    output_proc=self._ignore, json=False)


def _log_filter_addurls(res):
    return res.get('type') == 'file' and res.get('action') in ["addurl", "addurls"]


@with_result_progress("Adding URLs", log_filter=_log_filter_addurls)
def _add_urls(rows, ds, repo, ifexists=None, options=None,
              drop_after=False, by_key=False):
    """Call `git annex addurl` using information in `rows`.
    """
    add_url = partial(_add_url, ds=ds, repo=repo,
                      drop_after=drop_after, options=options)
    if by_key:
        # The by_key parameter isn't strictly needed, but it lets us avoid some
        # setup if --key wasn't specified.
        if repo.fake_dates_enabled:
            register_url = RegisterUrl(ds, repo)
        else:
            register_url = BatchedRegisterUrl(ds, repo)
    else:
        def register_url(*args, **kwargs):
            raise RuntimeError("bug: this should be impossible")

    add_metadata = {}
    for row in rows:
        filename_abs = row["filename_abs"]
        filename = row["ds_filename"]
        lgr.debug("Adding URLs to %s in %s", filename, ds.path)

        if os.path.exists(filename_abs) or os.path.islink(filename_abs):
            if ifexists == "skip":
                yield get_status_dict(action="addurls",
                                      ds=ds,
                                      type="file",
                                      path=filename_abs,
                                      status="notneeded")
                continue
            elif ifexists == "overwrite":
                lgr.debug("Removing %s", filename_abs)
                unlink(filename_abs)
            else:
                lgr.debug("File %s already exists", filename_abs)

        fn = register_url if row.get("key") else add_url
        all_ok = True
        for res in fn(row):
            if res["status"] != "ok":
                all_ok = False
            yield res
        if not all_ok:
            continue

        if row.get("meta_args"):
            add_metadata[filename] = row["meta_args"]

    if not add_metadata:
        return

    # For some reason interleaving batched addurl and regular metadata calls
    # causes multiple commits, so we do need to have it separate.
    # TODO: figure out with Joey either could be avoided
    from unittest.mock import patch
    with patch.object(repo, "always_commit", False):
        for filename, meta in add_metadata.items():
            lgr.debug("Adding metadata to %s in %s", filename, repo.path)
            assert not repo.always_commit
            for a in repo.set_metadata_(filename, add=meta):
                res = annexjson2result(a, ds, type="file", logger=lgr)
                if res["status"] == "ok":
                    # Don't show all added metadata for the file because that
                    # could quickly flood the output.
                    del res["message"]
                yield res


@build_doc
class Addurls(Interface):
    """Create and update a dataset from a list of URLs.

    *Format specification*

    Several arguments take format strings.  These are similar to normal Python
    format strings where the names from `URL-FILE` (column names for a comma-
    or tab-separated file or properties for JSON) are available as
    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can
    also be used (i.e., "{0}" for the first column). Note that a placeholder
    cannot contain a ':' or '!'.

    In addition, the `FILENAME-FORMAT` arguments has a few special
    placeholders.

      - _repindex

        The constructed file names must be unique across all fields rows.  To
        avoid collisions, the special placeholder "_repindex" can be added to
        the formatter.  Its value will start at 0 and increment every time a
        file name repeats.

      - _url_hostname, _urlN, _url_basename*

        Various parts of the formatted URL are available.  Take
        "http://datalad.org/asciicast/seamless_nested_repos.sh" as an example.

        "datalad.org" is stored as "_url_hostname".  Components of the URL's
        path can be referenced as "_urlN".  "_url0" and "_url1" would map to
        "asciicast" and "seamless_nested_repos.sh", respectively.  The final
        part of the path is also available as "_url_basename".

        This name is broken down further.  "_url_basename_root" and
        "_url_basename_ext" provide access to the root name and extension.
        These values are similar to the result of os.path.splitext, but, in the
        case of multiple periods, the extension is identified using the same
        length heuristic that git-annex uses.  As a result, the extension of
        "file.tar.gz" would be ".tar.gz", not ".gz".  In addition, the fields
        "_url_basename_root_py" and "_url_basename_ext_py" provide access to
        the result of os.path.splitext.

      - _url_filename*

        These are similar to _url_basename* fields, but they are obtained with
        a server request.  This is useful if the file name is set in the
        Content-Disposition header.


    *Examples*

    Consider a file "avatars.csv" that contains::

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and 'ext'
    fields, we could run::

      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'

    The `-d avatar_ds` is used to create a new dataset in "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new subdataset in an
    "avatars" subdirectory, we could use "//" in the `FILENAME-FORMAT`
    argument::

      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'

    If the information is represented as JSON lines instead of comma separated
    values or a JSON array, you can use a utility like jq to transform the JSON
    lines into an array that addurls accepts::

      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'

    .. note::

       For users familiar with 'git annex addurl': A large part of this
       plugin's functionality can be viewed as transforming data from
       `URL-FILE` into a "url filename" format that fed to 'git annex addurl
       --batch --with-files'.
    """

    from datalad.distribution.dataset import datasetmethod
    from datalad.interface.utils import eval_results
    from datalad.distribution.dataset import EnsureDataset
    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr
    from datalad.support.param import Parameter

    _params_ = dict(
        dataset=Parameter(
            args=("-d", "--dataset"),
            doc="""Add the URLs to this dataset (or possibly subdatasets of
            this dataset).  An empty or non-existent directory is passed to
            create a new dataset.  New subdatasets can be specified with
            `FILENAME-FORMAT`.""",
            constraints=EnsureDataset() | EnsureNone()),
        urlfile=Parameter(
            args=("urlfile",),
            metavar="URL-FILE",
            doc="""A file that contains URLs or information that can be used to
            construct URLs.  Depending on the value of --input-type, this
            should be a comma- or tab-separated file (with a header as the
            first row) or a JSON file (structured as a list of objects with
            string values). If '-', read from standard input, taking the
            content as JSON when --input-type is at its default value of
            'ext'."""),
        urlformat=Parameter(
            args=("urlformat",),
            metavar="URL-FORMAT",
            doc="""A format string that specifies the URL for each entry.  See
            the 'Format Specification' section above."""),
        filenameformat=Parameter(
            args=("filenameformat",),
            metavar="FILENAME-FORMAT",
            doc="""Like `URL-FORMAT`, but this format string specifies the file
            to which the URL's content will be downloaded. The name should be a
            relative path and will be taken as relative to the top-level
            dataset, regardless of whether it is specified via [PY: `dataset`
            PY][CMD: --dataset CMD] or inferred. The file name may contain
            directories. The separator "//" can be used to indicate that the
            left-side directory should be created as a new subdataset. See the
            'Format Specification' section above."""),
        input_type=Parameter(
            args=("-t", "--input-type"),
            metavar="TYPE",
            doc="""Whether `URL-FILE` should be considered a CSV file, TSV
            file, or JSON file. The default value, "ext", means to consider
            `URL-FILE` as a JSON file if it ends with ".json" or a TSV file if
            it ends with ".tsv". Otherwise, treat it as a CSV file.""",
            constraints=EnsureChoice(*INPUT_TYPES)),
        exclude_autometa=Parameter(
            args=("-x", "--exclude_autometa"),
            metavar="REGEXP",
            doc="""By default, metadata field=value pairs are constructed with
            each column in `URL-FILE`, excluding any single column that is
            specified via `URL-FORMAT`.  This argument can be used to exclude
            columns that match a regular expression.  If set to '*' or an empty
            string, automatic metadata extraction is disabled completely.  This
            argument does not affect metadata set explicitly with --meta."""),
        meta=Parameter(
            args=("-m", "--meta",),
            metavar="FORMAT",
            action="append",
            doc="""A format string that specifies metadata.  It should be
            structured as "<field>=<value>".  As an example, "location={3}"
            would mean that the value for the "location" metadata field should
            be set the value of the fourth column.  This option can be given
            multiple times."""),
        key=Parameter(
            args=("--key",),
            metavar="FORMAT",
            doc="""A format string that specifies an annex key for the file
            content. In this case, the file is not downloaded; instead the key
            is used to create the file without content. The value should be
            structured as "[et:]<input backend>[-s<bytes>]--<hash>". The
            optional "et:" prefix, which requires git-annex 8.20201116 or
            later, signals to toggle extension state of the input backend
            (i.e., MD5 vs MD5E). As an example, "et:MD5-s{size}--{md5sum}"
            would use the 'md5sum' and 'size' columns to construct the key,
            migrating the key from MD5 to MD5E, with an extension based on the
            file name. Note: If the *input* backend itself is an annex
            extension backend (i.e., a backend with a trailing "E"), the key's
            extension will not be updated to match the extension of the
            corresponding file name. Thus, unless the input keys and file names
            are generated from git-annex, it is recommended to avoid using
            extension backends as input. If an extension is desired, use the
            plain variant as input and prepend "et:" so that git-annex will
            migrate from the plain backend to the extension variant."""),
        message=Parameter(
            args=("--message",),
            metavar="MESSAGE",
            doc="""Use this message when committing the URL additions.""",
            constraints=EnsureNone() | EnsureStr()),
        dry_run=Parameter(
            args=("-n", "--dry-run"),
            action="store_true",
            doc="""Report which URLs would be downloaded to which files and
            then exit."""),
        fast=Parameter(
            args=("--fast",),
            action="store_true",
            doc="""If True, add the URLs, but don't download their content.
            Underneath, this passes the --fast flag to `git annex addurl`."""),
        ifexists=Parameter(
            args=("--ifexists",),
            doc="""What to do if a constructed file name already exists.  The
            default behavior is to proceed with the `git annex addurl`, which
            will fail if the file size has changed.  If set to 'overwrite',
            remove the old file before adding the new one.  If set to 'skip',
            do not add the new file.""",
            constraints=EnsureChoice(None, "overwrite", "skip")),
        missing_value=Parameter(
            args=("--missing-value",),
            metavar="VALUE",
            doc="""When an empty string is encountered, use this value
            instead.""",
            constraints=EnsureNone() | EnsureStr()),
        save=nosave_opt,
        version_urls=Parameter(
            args=("--version-urls",),
            action="store_true",
            doc="""Try to add a version ID to the URL. This currently only has
            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is
            not yet supported, but any URL that already contains a "versionId="
            parameter will be used as is."""),
        cfg_proc=Parameter(
            args=("-c", "--cfg-proc"),
            metavar="PROC",
            action='append',
            doc="""Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when
            calling `create` to make datasets."""),
        jobs=jobs_opt,
        drop_after=Parameter(
            args=("--drop-after",),
            action="store_true",
            doc="""drop files after adding to annex""",
        ),
    )

    @staticmethod
    @datasetmethod(name='addurls')
    @eval_results
    def __call__(dataset, urlfile, urlformat, filenameformat,
                 input_type="ext", exclude_autometa=None, meta=None, key=None,
                 message=None, dry_run=False, fast=False, ifexists=None,
                 missing_value=None, save=True, version_urls=False,
                 cfg_proc=None, jobs=None, drop_after=False):
        # Temporarily work around gh-2269.
        url_file = urlfile
        url_format, filename_format = urlformat, filenameformat

        from requests.exceptions import RequestException

        from datalad.distribution.dataset import Dataset, require_dataset
        from datalad.support.annexrepo import AnnexRepo

        lgr = logging.getLogger("datalad.plugin.addurls")

        ds = require_dataset(dataset, check_installed=False)
        repo = ds.repo
        st_dict = get_status_dict(action="addurls", ds=ds)
        if repo and not isinstance(repo, AnnexRepo):
            yield dict(st_dict, status="error", message="not an annex repo")
            return

        if key:
            old_examinekey = external_versions["cmd:annex"] < "8.20201116"
            if old_examinekey:
                old_msg = None
                if key.startswith("et:"):
                    old_msg = ("et: prefix of `key` option requires "
                               "git-annex 8.20201116 or later")
                elif repo.is_managed_branch():
                    old_msg = ("Using `key` option on adjusted branch "
                               "requires git-annex 8.20201116 or later")
                if old_msg:
                    yield dict(st_dict, status="error", message=old_msg)
                    return

        if url_file != "-":
            url_file = str(resolve_path(url_file, dataset))

        if input_type == "ext":
            if url_file == "-":
                input_type = "json"
            else:
                extension = os.path.splitext(url_file)[1]
                if extension == ".json":
                    input_type = "json"
                elif extension == ".tsv":
                    input_type = "tsv"
                else:
                    input_type = "csv"

        fd = sys.stdin if url_file == "-" else open(url_file)
        try:
            try:
                rows, subpaths = extract(fd, input_type,
                                         url_format, filename_format,
                                         exclude_autometa, meta, key,
                                         dry_run,
                                         missing_value)
            except (ValueError, RequestException) as exc:
                yield dict(st_dict, status="error", message=exc_str(exc))
                return
        finally:
            if fd is not sys.stdin:
                fd.close()

        if not rows:
            yield dict(st_dict, status="notneeded",
                       message="No rows to process")
            return

        if len(rows) != len(set(row["filename"] for row in rows)):
            yield dict(st_dict, status="error",
                       message=("There are file name collisions; "
                                "consider using {_repindex}"))
            return

        if dry_run:
            for subpath in subpaths:
                lgr.info("Would create a subdataset at %s", subpath)
            for row in rows:
                lgr.info("Would %s %s to %s",
                         "register" if row.get("key") else "download",
                         row["url"],
                         os.path.join(ds.path, row["filename"]))
                if "meta_args" in row:
                    lgr.info("Metadata: %s",
                             sorted(u"{}={}".format(k, v)
                                    for k, v in row["meta_args"].items()))
            yield dict(st_dict, status="ok", message="dry-run finished")
            return

        if not repo:
            # Populate a new dataset with the URLs.
            yield from ds.create(
                result_xfm=None,
                return_type='generator',
                cfg_proc=cfg_proc)

        annex_options = ["--fast"] if fast else []

        # to be populated by addurls_to_ds
        files_to_add = set()
        created_subds = []

        def addurls_to_ds(args):
            """The "consumer" for ProducerConsumer parallel execution"""
            subpath, rows = args

            ds_path = ds.path  # shortcut on closure from outside

            # technically speaking, subds might be the ds
            if subpath:
                subds_path = os.path.join(ds_path, subpath)
            else:
                subds_path = ds_path

            for row in rows:
                # Add additional information that we'll need for various
                # operations.
                filename_abs = op.join(ds_path, row["filename"])
                ds_filename = op.relpath(filename_abs, subds_path)
                row.update({"filename_abs": filename_abs,
                            "ds_filename": ds_filename})

            subds = Dataset(subds_path)

            if subds.is_installed():
                lgr.debug(
                    "Not creating subdataset at existing path: %s",
                    subds_path)
            else:
                yield from subds.create(result_xfm=None,
                                        cfg_proc=cfg_proc,
                                        return_type='generator')
                created_subds.append(subpath)
            repo = subds.repo  # "expensive" so we get it once

            if version_urls:
                num_urls = len(rows)
                log_progress(lgr.info, "addurls_versionurls",
                             "Versioning %d URLs", num_urls,
                             label="Versioning URLs",
                             total=num_urls, unit=" URLs")
                for row in rows:
                    url = row["url"]
                    try:
                        # TODO: make get_versioned_url more efficient while going
                        # through the same bucket(s)
                        row["url"] = get_versioned_url(url)
                    except (ValueError, NotImplementedError) as exc:
                        # We don't expect this to happen because get_versioned_url
                        # should return the original URL if it isn't an S3 bucket.
                        # It only raises exceptions if it doesn't know how to
                        # handle the scheme for what looks like an S3 bucket.
                        lgr.warning("error getting version of %s: %s",
                                    row["url"], exc_str(exc))
                    log_progress(lgr.info, "addurls_versionurls",
                                 "Versioned result for %s: %s", url, row["url"],
                                 update=1, increment=True)
                log_progress(lgr.info, "addurls_versionurls", "Finished versioning URLs")

            subds_files_to_add = set()
            for r in _add_urls(rows, subds, repo,
                               ifexists=ifexists, options=annex_options,
                               drop_after=drop_after, by_key=key):
                if r["status"] == "ok":
                    subds_files_to_add.add(r["path"])
                yield r

            files_to_add.update(subds_files_to_add)
            pass  # end of addurls_to_ds


        # We need to group by dataset since otherwise we will initiate
        # batched annex process per each subdataset, which might be infeasible
        # in any use-case with a considerable number of subdatasets.
        # Also groupping allows us for parallelization across datasets, and avoids
        # proliferation of commit messages upon creation of each individual subdataset.

        def keyfn(d):
            # The top-level dataset has a subpath of None.
            return d.get("subpath") or ""

        # We need to serialize itertools.groupby .
        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]

        # There could be "intermediate" subdatasets which have no rows but would need
        # their datasets created and saved, so let's add them
        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)
        nrows_by_ds_orig = len(rows_by_ds)
        if add_subpaths:
            for subpath in add_subpaths:
                rows_by_ds.append((subpath, tuple()))
            # and now resort them again since we added
            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])

        # We want to provide progress overall files not just datasets
        # so our total will be just a len of rows
        def agg_files(*args, **kwargs):
            return len(rows)

        yield from ProducerConsumerProgressLog(
            rows_by_ds,
            addurls_to_ds,
            agg=agg_files,
            # It is ok to start with subdatasets since top dataset already exists
            safe_to_consume=partial(no_parentds_in_futures, skip=("", None, ".")),
            # our producer provides not only dataset paths and also rows, take just path
            producer_future_key=lambda row_by_ds: row_by_ds[0],
            jobs=jobs,
            # Logging options
            # we will be yielding all kinds of records, but of interest for progress
            # reporting only addurls on files
            # note: annexjson2result overrides 'action' with 'command' content from
            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.
            # TODO: see if it is all ok, since we might be now yielding both
            # addurls and addurl records.
            log_filter=_log_filter_addurls,
            unit="files",
            lgr=lgr,
        )

        if save:
            extra_msgs = []
            if created_subds:
                extra_msgs.append(f"{len(created_subds)} subdatasets were created")
            if extra_msgs:
                extra_msgs.append('')
            message_addurls = message or f"""\
[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs

{os.linesep.join(extra_msgs)}
url_file='{urlfile}'
url_format='{url_format}'
filename_format='{filenameformat}'"""

            # save all in bulk
            files_to_add.update([r[0] for r in rows_by_ds])
            yield from ds.save(
                list(files_to_add),
                message=message_addurls,
                jobs=jobs,
                return_type='generator')


__datalad_plugin__ = Addurls

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Extract Function Fails in Tests Due to Missing `input_type` Parameter

**Description:**
After recent changes to the `extract` function to support passing a list of records, several tests are failing because they still call `extract` without the required `input_type` argument.

**Example Buggy Code:**
```python
info, subpaths = au.extract(
    ST_DATA['rows'],
    url_format='{name}_{debut_season}.com',
    filename_format='{age_group}//{now_dead}//{name}.csv'
)
```

**Expected Behavior:**
The `extract` function should process the provided records correctly without requiring the `input_type` parameter, accommodating the new functionality of accepting a list of records.

**Actual Behavior:**
Calling `extract` without the `input_type` parameter results in the following error:
```
TypeError: extract() missing 1 required positional argument: 'input_type'
```
This prevents the tests from executing successfully and indicates a mismatch between the function's updated signature and its usage in the tests.
[/ISSUE]
{
    "test_formatter": "PASSED",
    "test_formatter_lower_case": "PASSED",
    "test_formatter_no_idx_map": "PASSED",
    "test_formatter_no_mapping_arg": "PASSED",
    "test_formatter_placeholder_with_spaces": "PASSED",
    "test_formatter_placeholder_nonpermitted_chars": "PASSED",
    "test_formatter_missing_arg": "PASSED",
    "test_repformatter": "PASSED",
    "test_clean_meta_args": "PASSED",
    "test_get_subpaths": "PASSED",
    "test_sort_paths": "PASSED",
    "test_is_legal_metafield": "PASSED",
    "test_filter_legal_metafield": "PASSED",
    "test_fmt_to_name": "PASSED",
    "test_get_file_parts": "PASSED",
    "test_get_url_parts": "PASSED",
    "test_extract": "PASSED",
    "test_extract_disable_autometa": "PASSED",
    "test_extract_exclude_autometa_regexp": "PASSED",
    "test_extract_wrong_input_type": "PASSED",
    "test_addurls_nonannex_repo": "FAILED",
    "test_addurls_unknown_placeholder": "FAILED",
    "test_addurls_dry_run": "FAILED",
    "TestAddurls.test_addurls": "FAILED",
    "TestAddurls.test_addurls_unbound_dataset": "FAILED",
    "TestAddurls.test_addurls_create_newdataset": "FAILED",
    "TestAddurls.test_addurls_from_list": "FAILED",
    "TestAddurls.test_addurls_subdataset": "FAILED",
    "TestAddurls.test_addurls_repindex": "FAILED",
    "TestAddurls.test_addurls_url_parts": "FAILED",
    "TestAddurls.test_addurls_url_filename": "FAILED",
    "TestAddurls.test_addurls_url_filename_fail": "FAILED",
    "TestAddurls.test_addurls_metafail": "FAILED",
    "TestAddurls.test_addurls_dropped_urls": "FAILED",
    "TestAddurls.test_addurls_version": "FAILED",
    "TestAddurls.test_addurls_deeper": "FAILED",
    "TestAddurls.test_addurls_invalid_input": "FAILED",
    "TestAddurls.test_addurls_no_rows": "FAILED",
    "TestAddurls.test_addurls_stdin_input_command_line": "FAILED",
    "TestAddurls.test_drop_after": "FAILED",
    "TestAddurls.test_addurls_from_key_invalid_format": "FAILED",
    "TestAddurls.test_addurls_row_missing_key_fields": "FAILED"
}
diff --git a/datalad/plugin/addurls.py b/datalad/plugin/addurls.py
index 1fe43d733..da56bf64f 100644
--- a/datalad/plugin/addurls.py
+++ b/datalad/plugin/addurls.py
@@ -399,6 +399,31 @@ def _read(stream, input_type):
     return rows, idx_map
 
 
+def _read_from_file(fname, input_type):
+    from_stdin = fname == "-"
+    if input_type == "ext":
+        if from_stdin:
+            input_type = "json"
+        else:
+            extension = os.path.splitext(fname)[1]
+            if extension == ".json":
+                input_type = "json"
+            elif extension == ".tsv":
+                input_type = "tsv"
+            else:
+                input_type = "csv"
+
+    fd = sys.stdin if from_stdin else open(fname)
+    try:
+        records, colidx_to_name = _read(fd, input_type)
+        if not records:
+            lgr.warning("No rows found in %s", fd)
+    finally:
+        if fd is not sys.stdin:
+            fd.close()
+    return records, colidx_to_name
+
+
 def _get_placeholder_exception(exc, msg_prefix, known):
     """Recast KeyError as a ValueError with close-match suggestions.
     """
@@ -540,16 +565,17 @@ def sort_paths(paths):
     yield from sorted(paths, key=level_and_name)
 
 
-def extract(stream, input_type, url_format="{0}", filename_format="{1}",
+def extract(rows, colidx_to_name=None,
+            url_format="{0}", filename_format="{1}",
             exclude_autometa=None, meta=None, key=None,
             dry_run=False, missing_value=None):
-    """Extract and format information from `url_file`.
+    """Extract and format information from `rows`.
 
     Parameters
     ----------
-    stream : file object
-        Items used to construct the file names and URLs.
-    input_type : {'csv', 'tsv', 'json'}
+    rows : list of dict
+    colidx_to_name : dict, optional
+        Mapping from a position index to a column name.
 
     All other parameters match those described in `AddUrls`.
 
@@ -560,11 +586,7 @@ def extract(stream, input_type, url_format="{0}", filename_format="{1}",
     sorted breadth-first.
     """
     meta = ensure_list(meta)
-
-    rows, colidx_to_name = _read(stream, input_type)
-    if not rows:
-        lgr.warning("No rows found in %s", stream)
-        return [], []
+    colidx_to_name = colidx_to_name or {}
 
     # Formatter for everything but file names
     fmt = Formatter(colidx_to_name, missing_value)
@@ -978,7 +1000,8 @@ class Addurls(Interface):
             first row) or a JSON file (structured as a list of objects with
             string values). If '-', read from standard input, taking the
             content as JSON when --input-type is at its default value of
-            'ext'."""),
+            'ext'. [PY:  Alternatively, an iterable of dicts can be given.
+            PY]"""),
         urlformat=Parameter(
             args=("urlformat",),
             metavar="URL-FORMAT",
@@ -1132,25 +1155,28 @@ class Addurls(Interface):
                     yield dict(st_dict, status="error", message=old_msg)
                     return
 
-        if url_file != "-":
-            url_file = str(resolve_path(url_file, dataset))
-
-        if input_type == "ext":
-            if url_file == "-":
-                input_type = "json"
-            else:
-                extension = os.path.splitext(url_file)[1]
-                if extension == ".json":
-                    input_type = "json"
-                elif extension == ".tsv":
-                    input_type = "tsv"
-                else:
-                    input_type = "csv"
+        if isinstance(url_file, str):
+            if url_file != "-":
+                url_file = str(resolve_path(url_file, dataset))
+            try:
+                records, colidx_to_name = _read_from_file(
+                    url_file, input_type)
+            except ValueError as exc:
+                yield get_status_dict(action="addurls",
+                                      ds=ds,
+                                      status="error",
+                                      message=exc_str(exc))
+                return
+            displayed_source = "'{}'".format(urlfile)
+        else:
+            displayed_source = "<records>"
+            records = ensure_list(url_file)
+            colidx_to_name = {}
 
-        fd = sys.stdin if url_file == "-" else open(url_file)
-        try:
+        rows = None
+        if records:
             try:
-                rows, subpaths = extract(fd, input_type,
+                rows, subpaths = extract(records, colidx_to_name,
                                          url_format, filename_format,
                                          exclude_autometa, meta, key,
                                          dry_run,
@@ -1158,9 +1184,6 @@ class Addurls(Interface):
             except (ValueError, RequestException) as exc:
                 yield dict(st_dict, status="error", message=exc_str(exc))
                 return
-        finally:
-            if fd is not sys.stdin:
-                fd.close()
 
         if not rows:
             yield dict(st_dict, status="notneeded",
@@ -1329,7 +1352,7 @@ class Addurls(Interface):
 [DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs
 
 {os.linesep.join(extra_msgs)}
-url_file='{urlfile}'
+url_file={displayed_source}
 url_format='{url_format}'
 filename_format='{filenameformat}'"""
 

{"datalad/plugin/addurls.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Create and update a dataset from a list of URLs.\n\"\"\"\n\nfrom collections.abc import Mapping\n\nimport logging\nimport os\nimport re\nimport string\nimport sys\n\nfrom functools import partial\nfrom urllib.parse import urlparse\n\nfrom datalad.support.external_versions import external_versions\nimport datalad.support.path as op\nfrom datalad.distribution.dataset import resolve_path\nfrom datalad.dochelpers import exc_str\nfrom datalad.log import log_progress, with_result_progress\nfrom datalad.interface.base import Interface\nfrom datalad.interface.base import build_doc\nfrom datalad.interface.results import annexjson2result, get_status_dict\nfrom datalad.interface.common_opts import (\n    jobs_opt,\n    nosave_opt,\n)\nfrom datalad.support.exceptions import CommandError\nfrom datalad.support.itertools import groupby_sorted\nfrom datalad.support.network import get_url_filename\nfrom datalad.support.path import split_ext\nfrom datalad.support.parallel import (\n    ProducerConsumerProgressLog,\n    no_parentds_in_futures,\n)\nfrom datalad.support.s3 import get_versioned_url\nfrom datalad.utils import (\n    ensure_list,\n    get_suggestions_msg,\n    Path,\n    unlink,\n)\n\nlgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n__docformat__ = \"restructuredtext\"\n\n\nclass Formatter(string.Formatter):\n    \"\"\"Formatter that gives precedence to custom keys.\n\n    The first positional argument to the `format` call should be a\n    mapping whose keys are exposed as placeholders (e.g.,\n    \"{key1}.py\").\n\n    Parameters\n    ----------\n    idx_to_name : dict\n        A mapping from a positional index to a key.  If not provided,\n        \"{N}\" elements are not supported.\n    missing : str, optional\n        When column lookup results in an empty string, use this value in\n        its place.\n    \"\"\"\n\n    def __init__(self, idx_to_name=None, missing_value=None):\n        self.idx_to_name = idx_to_name or {}\n        self.missing = missing_value\n\n    def format(self, format_string, *args, **kwargs):\n        if not isinstance(args[0], Mapping):\n            raise ValueError(\"First positional argument should be mapping\")\n        return super(Formatter, self).format(format_string, *args, **kwargs)\n\n    def get_value(self, key, args, kwargs):\n        \"\"\"Look for key's value in `args[0]` mapping first.\n        \"\"\"\n        # FIXME: This approach will fail for keys that contain \"!\" and\n        # \":\" because they'll be interpreted as formatting flags.\n        data = args[0]\n\n        name = key\n        try:\n            key_int = int(key)\n        except ValueError:\n            pass\n        else:\n            name = self.idx_to_name[key_int]\n\n        try:\n            value = data[name]\n        except KeyError:\n            return super(Formatter, self).get_value(\n                key, args, kwargs)\n\n        if self.missing is not None and isinstance(value, str):\n            return value or self.missing\n        return value\n\n    def convert_field(self, value, conversion):\n        if conversion == 'l':\n            return str(value).lower()\n        return super(Formatter, self).convert_field(value, conversion)\n\n\nclass RepFormatter(Formatter):\n    \"\"\"Extend Formatter to support a {_repindex} placeholder.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(RepFormatter, self).__init__(*args, **kwargs)\n        self.repeats = {}\n        self.repindex = 0\n\n    def format(self, *args, **kwargs):\n        self.repindex = 0\n        result = super(RepFormatter, self).format(*args, **kwargs)\n        if result in self.repeats:\n            self.repindex = self.repeats[result] + 1\n            self.repeats[result] = self.repindex\n            result = super(RepFormatter, self).format(*args, **kwargs)\n        else:\n            self.repeats[result] = 0\n        return result\n\n    def get_value(self, key, args, kwargs):\n        args[0][\"_repindex\"] = self.repindex\n        return super(RepFormatter, self).get_value(key, args, kwargs)\n\n\ndef clean_meta_args(args):\n    \"\"\"Process metadata arguments.\n\n    Parameters\n    ----------\n    args : iterable of str\n        Formatted metadata arguments for 'git-annex metadata --set'.\n\n    Returns\n    -------\n    A dict mapping field names to values.\n    \"\"\"\n    results = {}\n    for arg in args:\n        parts = [x.strip() for x in arg.split(\"=\", 1)]\n        if len(parts) == 2:\n            if not parts[0]:\n                raise ValueError(\"Empty field name\")\n            field, value = parts\n        else:\n            raise ValueError(\"meta argument isn't in 'field=value' format\")\n\n        if not value:\n            # The `url_file` may have an empty value.\n            continue\n        results[field] = value\n    return results\n\n\ndef get_subpaths(filename):\n    \"\"\"Convert \"//\" marker in `filename` to a list of subpaths.\n\n    >>> from datalad.plugin.addurls import get_subpaths\n    >>> get_subpaths(\"p1/p2//p3/p4//file\")\n    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])\n\n    Note: With Python 3, the subpaths could be generated with\n\n        itertools.accumulate(filename.split(\"//\")[:-1], os.path.join)\n\n    Parameters\n    ----------\n    filename : str\n        File name with \"//\" marking subpaths.\n\n    Returns\n    -------\n    A tuple of the filename with any \"//\" collapsed to a single\n    separator and a list of subpaths (str).\n    \"\"\"\n    if \"//\" not in filename:\n        return filename, []\n\n    spaths = []\n    for part in filename.split(\"//\")[:-1]:\n        path = os.path.join(spaths[-1], part) if spaths else part\n        spaths.append(path)\n    return filename.replace(\"//\", os.path.sep), spaths\n\n\ndef is_legal_metafield(name):\n    \"\"\"Test whether `name` is a valid metadata field.\n\n    The set of permitted characters is taken from git-annex's\n    MetaData.hs:legalField.\n    \"\"\"\n    return bool(re.match(r\"[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z\", name))\n\n\ndef filter_legal_metafield(fields):\n    \"\"\"Remove illegal names from `fields`.\n\n    Note: This is like `filter(is_legal_metafield, fields)` but the\n    dropped values are logged.\n    \"\"\"\n    legal = []\n    for field in fields:\n        if is_legal_metafield(field):\n            legal.append(field)\n        else:\n            lgr.debug(\"%s is not a valid metadata field name; dropping\",\n                      field)\n    return legal\n\n\nclass AnnexKeyParser(object):\n    \"\"\"Parse a full annex key into subparts.\n\n    The key may have an \"et:\" prefix appended, which signals that the backend's\n    extension state should be toggled.\n\n    See <https://git-annex.branchable.com/internals/key_format/>.\n\n    Parameters\n    ----------\n    format_fn : callable\n        Function that takes a format string and a row and returns the full key.\n    format_string : str\n        Format string for the full key.\n    \"\"\"\n\n    def __init__(self, format_fn, format_string):\n        self.format_fn = format_fn\n        self.format_string = format_string\n        self.regexp = re.compile(r\"(?P<et>et:)?\"\n                                 r\"(?P<backend>[A-Z0-9]+)\"\n                                 r\"(?:-[^-]+)?\"\n                                 r\"--(?P<keyname>[^/\\n]+)\\Z\")\n        self.empty = \"\".join(i[0]\n                             for i in string.Formatter().parse(format_string))\n\n    @staticmethod\n    def _validate_common(backend, key):\n        if backend.endswith(\"E\"):\n            ext = True\n            backend = backend[:-1]\n        else:\n            ext = False\n\n        expected_lengths = {\"MD5\": 32,\n                            \"SHA1\": 40,\n                            \"SHA256\": 64}\n\n        expected_len = expected_lengths.get(backend)\n        if not expected_len:\n            return\n\n        if ext and \".\" in key:\n            key = key.split(\".\", maxsplit=1)[0]\n\n        if len(key) != expected_len:\n            raise ValueError(\"{} key does not have expected length of {}: {}\"\n                             .format(backend, expected_len, key))\n\n        if not re.match(r\"[a-f0-9]+\\Z\", key):\n            raise ValueError(\"{} key has non-hexadecimal character: {}\"\n                             .format(backend, key))\n\n    def parse(self, row):\n        \"\"\"Format the key with the fields in `row` and parse it.\n\n        Returns\n        -------\n        A dictionary with the following keys that match their counterparts in\n        the output of `git annex examinekey --json`: \"key\" (the full annex\n        key), \"backend\", and \"keyname\". If the key had an \"et:\" prefix, there\n        is also a \"target_backend\" key.\n\n        Raises\n        ------\n        ValueError if the formatted value doesn't look like a valid key\n        \"\"\"\n        try:\n            key = self.format_fn(self.format_string, row)\n        except KeyError as exc:\n            lgr.debug(\"Row missing fields for --key: %s\",\n                      exc_str(exc))\n            return {}\n\n        if key == self.empty:\n            lgr.debug(\"All fields in --key's value are empty in row: %s\", row)\n            # We got the same string that'd we get if all the fields were\n            # empty, so this doesn't have a key.\n            return {}\n\n        match = self.regexp.match(key)\n        if match:\n            info = match.groupdict()\n            et = info.pop(\"et\", None)\n            if et:\n                info[\"key\"] = key[3:]  # Drop \"et:\" from full key.\n                backend = info[\"backend\"]\n                if backend.endswith(\"E\"):\n                    info[\"target_backend\"] = backend[:-1]\n                else:\n                    info[\"target_backend\"] = backend + \"E\"\n            else:\n                info[\"key\"] = key\n            self._validate_common(info[\"backend\"], info[\"keyname\"])\n            return info\n        else:\n            raise ValueError(\n                \"Key does not match expected \"\n                \"[et:]<backend>[-[CmsS]NNN]--<key> format: {}\"\n                .format(key))\n\n\ndef get_fmt_names(format_string):\n    \"\"\"Yield field names in `format_string`.\n    \"\"\"\n    for _, name, _, _ in string.Formatter().parse(format_string):\n        if name:\n            yield name\n\n\ndef fmt_to_name(format_string, num_to_name):\n    \"\"\"Try to map a format string to a single name.\n\n    Parameters\n    ----------\n    format_string : string\n    num_to_name : dict\n        A dictionary that maps from an integer to a column name.  This\n        enables mapping the format string to an integer to a name.\n\n    Returns\n    -------\n    A placeholder name if `format_string` consists of a single\n    placeholder and no other text.  Otherwise, None is returned.\n    \"\"\"\n    parsed = list(string.Formatter().parse(format_string))\n    if len(parsed) != 1:\n        # It's an empty string or there's more than one placeholder.\n        return\n    if parsed[0][0]:\n        # Format string contains text before the placeholder.\n        return\n\n    name = parsed[0][1]\n    if not name:\n        # The field name is empty.\n        return\n\n    try:\n        return num_to_name[int(name)]\n    except (KeyError, ValueError):\n        return name\n\n\nINPUT_TYPES = [\"ext\", \"csv\", \"tsv\", \"json\"]\n\n\ndef _read(stream, input_type):\n    if input_type in [\"csv\", \"tsv\"]:\n        import csv\n        csvrows = csv.reader(stream,\n                             delimiter=\"\\t\" if input_type == \"tsv\" else \",\")\n        try:\n            headers = next(csvrows)\n        except StopIteration:\n            raise ValueError(\"Failed to read {} rows from {}\"\n                             .format(input_type.upper(), stream))\n        lgr.debug(\"Taking %s fields from first line as headers: %s\",\n                  len(headers), headers)\n        idx_map = dict(enumerate(headers))\n        rows = [dict(zip(headers, r)) for r in csvrows]\n    elif input_type == \"json\":\n        import json\n        try:\n            rows = json.load(stream)\n        except json.decoder.JSONDecodeError as e:\n            raise ValueError(\n                \"Failed to read JSON from stream {}: {}\"\n                .format(stream, exc_str(e)))\n        # For json input, we do not support indexing by position,\n        # only names.\n        idx_map = {}\n    else:\n        raise ValueError(\n            \"input_type {} is invalid. Known values: {}\"\n            .format(input_type, \", \".join(INPUT_TYPES)))\n    return rows, idx_map\n\n\ndef _get_placeholder_exception(exc, msg_prefix, known):\n    \"\"\"Recast KeyError as a ValueError with close-match suggestions.\n    \"\"\"\n    value = exc.args[0]\n    if isinstance(value, str):\n        sugmsg = get_suggestions_msg(value, known)\n    else:\n        sugmsg = \"Out-of-bounds or unsupported index.\"\n    # Note: Keeping this a KeyError is probably more appropriate but then the\n    # entire message, which KeyError takes as the key, will be rendered with\n    # outer quotes.\n    return ValueError(\"{}: {}{}{}\"\n                      .format(msg_prefix, exc, \". \" if sugmsg else \"\", sugmsg))\n\n\ndef _format_filenames(format_fn, rows, row_infos):\n    subpaths = set()\n    for row, info in zip(rows, row_infos):\n        try:\n            filename = format_fn(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in file name\", row)\n        filename, spaths = get_subpaths(filename)\n        subpaths |= set(spaths)\n        info[\"filename\"] = filename\n        info[\"subpath\"] = spaths[-1] if spaths else None\n    return subpaths\n\n\ndef get_file_parts(filename, prefix=\"name\"):\n    \"\"\"Assign a name to various parts of a file.\n\n    Parameters\n    ----------\n    filename : str\n        A file name (no leading path is permitted).\n    prefix : str\n        Prefix to prepend to the key names.\n\n    Returns\n    -------\n    A dict mapping each part to a value.\n    \"\"\"\n    root, ext = split_ext(filename)\n    root_py, ext_py = os.path.splitext(filename)\n\n    return {prefix: filename,\n            prefix + \"_root\": root,\n            prefix + \"_ext\": ext,\n            prefix + \"_root_py\": root_py,\n            prefix + \"_ext_py\": ext_py}\n\n\ndef get_url_parts(url):\n    \"\"\"Assign a name to various parts of the URL.\n\n    Parameters\n    ----------\n    url : str\n\n    Returns\n    -------\n    A dict with keys `_url_hostname` and, for a path with N+1 parts,\n    '_url0' through '_urlN' .  There is also a `_url_basename` key for\n    the rightmost part of the path.\n    \"\"\"\n    parsed = urlparse(url)\n    if not parsed.netloc:\n        return {}\n\n    names = {\"_url_hostname\": parsed.netloc}\n\n    path = parsed.path.strip(\"/\")\n    if not path:\n        return names\n\n    url_parts = path.split(\"/\")\n    for pidx, part in enumerate(url_parts):\n        names[\"_url{}\".format(pidx)] = part\n    basename = url_parts[-1]\n    names[\"_url_basename\"] = basename\n    names.update(get_file_parts(basename, prefix=\"_url_basename\"))\n    return names\n\n\ndef add_extra_filename_values(filename_format, rows, urls, dry_run):\n    \"\"\"Extend `rows` with values for special formatting fields.\n    \"\"\"\n    file_fields = list(get_fmt_names(filename_format))\n    if any(i.startswith(\"_url\") for i in file_fields):\n        for row, url in zip(rows, urls):\n            row.update(get_url_parts(url))\n\n    if any(i.startswith(\"_url_filename\") for i in file_fields):\n        if dry_run:  # Don't waste time making requests.\n            dummy = get_file_parts(\"BASE.EXT\", \"_url_filename\")\n            for idx, row in enumerate(rows):\n                row.update(\n                    {k: v + str(idx) for k, v in dummy.items()})\n        else:\n            num_urls = len(urls)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Requesting file names for %d URLs\", num_urls,\n                         label=\"Requesting names\", total=num_urls,\n                         unit=\" Files\")\n            for row, url in zip(rows, urls):\n                # If we run into any issues here, we're just going to raise an\n                # exception and then abort inside dlplugin.  It'd be good to\n                # disentangle this from `extract` so that we could yield an\n                # individual error, drop the row, and keep going.\n                filename = get_url_filename(url)\n                if filename:\n                    row.update(get_file_parts(filename, \"_url_filename\"))\n                else:\n                    raise ValueError(\n                        \"{} does not contain a filename\".format(url))\n                log_progress(lgr.info, \"addurls_requestnames\",\n                             \"%s returned for %s\", url, filename,\n                             update=1, increment=True)\n            log_progress(lgr.info, \"addurls_requestnames\",\n                         \"Finished requesting file names\")\n\n\ndef sort_paths(paths):\n    \"\"\"Sort `paths` by directory level and then alphabetically.\n\n    Parameters\n    ----------\n    paths : iterable of str\n\n    Returns\n    -------\n    Generator of sorted paths.\n    \"\"\"\n    def level_and_name(p):\n        return p.count(os.path.sep), p\n\n    yield from sorted(paths, key=level_and_name)\n\n\ndef extract(stream, input_type, url_format=\"{0}\", filename_format=\"{1}\",\n            exclude_autometa=None, meta=None, key=None,\n            dry_run=False, missing_value=None):\n    \"\"\"Extract and format information from `url_file`.\n\n    Parameters\n    ----------\n    stream : file object\n        Items used to construct the file names and URLs.\n    input_type : {'csv', 'tsv', 'json'}\n\n    All other parameters match those described in `AddUrls`.\n\n    Returns\n    -------\n    A tuple where the first item is a list with a dict of extracted information\n    for each row in `stream` and the second item a list subdataset paths,\n    sorted breadth-first.\n    \"\"\"\n    meta = ensure_list(meta)\n\n    rows, colidx_to_name = _read(stream, input_type)\n    if not rows:\n        lgr.warning(\"No rows found in %s\", stream)\n        return [], []\n\n    # Formatter for everything but file names\n    fmt = Formatter(colidx_to_name, missing_value)\n    format_url = partial(fmt.format, url_format)\n\n    auto_meta_args = []\n    if exclude_autometa not in [\"*\", \"\"]:\n        urlcol = fmt_to_name(url_format, colidx_to_name)\n        # TODO: Try to normalize invalid fields, checking for any\n        # collisions.\n        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)\n        if exclude_autometa:\n            metacols = (c for c in metacols\n                        if not re.search(exclude_autometa, c))\n        metacols = filter_legal_metafield(metacols)\n        auto_meta_args = [c + \"=\" + \"{\" + c + \"}\" for c in metacols]\n\n    # Unlike `filename_format` and `url_format`, `meta` is a list\n    # because meta may be given multiple times on the command line.\n    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]\n\n    info_fns = []\n    if formats_meta:\n        def set_meta_args(info, row):\n            info[\"meta_args\"] = clean_meta_args(fmt(row)\n                                                for fmt in formats_meta)\n        info_fns.append(set_meta_args)\n    if key:\n        key_parser = AnnexKeyParser(fmt.format, key)\n\n        def set_key(info, row):\n            info[\"key\"] = key_parser.parse(row)\n        info_fns.append(set_key)\n\n    rows_with_url = []\n    infos = []\n    for row in rows:\n        try:\n            url = format_url(row)\n        except KeyError as exc:\n            raise _get_placeholder_exception(\n                exc, \"Unknown placeholder in URL\", row)\n        if not url or url == missing_value:\n            continue  # pragma: no cover, peephole optimization\n        rows_with_url.append(row)\n        info = {\"url\": url}\n        for fn in info_fns:\n            fn(info, row)\n        infos.append(info)\n\n    n_dropped = len(rows) - len(rows_with_url)\n    if n_dropped:\n        lgr.warning(\"Dropped %d row(s) that had an empty URL\", n_dropped)\n\n    # Format the filename in a second pass so that we can provide\n    # information about the formatted URLs.\n    add_extra_filename_values(filename_format, rows_with_url,\n                              [i[\"url\"] for i in infos],\n                              dry_run)\n\n    # For the file name, we allow the _repindex special key.\n    format_filename = partial(\n        RepFormatter(colidx_to_name, missing_value).format,\n        filename_format)\n    subpaths = _format_filenames(format_filename, rows_with_url, infos)\n    return infos, list(sort_paths(subpaths))\n\n\ndef _add_url(row, ds, repo, options=None, drop_after=False):\n    filename_abs = row[\"filename_abs\"]\n    filename = row[\"ds_filename\"]\n    try:\n        out_json = repo.add_url_to_file(filename, row[\"url\"],\n                                        batch=True, options=options)\n    except CommandError as exc:\n        yield get_status_dict(action=\"addurls\",\n                              ds=ds,\n                              type=\"file\",\n                              path=filename_abs,\n                              message=exc_str(exc),\n                              status=\"error\")\n        return\n\n    # In the case of an error, the json object has file=None.\n    if out_json[\"file\"] is None:\n        out_json[\"file\"] = filename_abs\n    res_addurls = annexjson2result(\n        out_json, ds, action=\"addurls\",\n        type=\"file\", logger=lgr)\n    yield res_addurls\n\n    if not res_addurls[\"status\"] == \"ok\":\n        return\n\n    if drop_after and 'annexkey' in res_addurls:\n        # unfortunately .drop has no batched mode, and drop_key ATM would\n        # raise AssertionError if not success, and otherwise return nothing\n        try:\n            repo.drop_key(res_addurls['annexkey'], batch=True)\n            st_kwargs = dict(status=\"ok\")\n        except (AssertionError, CommandError) as exc:\n            st_kwargs = dict(message=exc_str(exc),\n                             status=\"error\")\n        yield get_status_dict(action=\"drop\",\n                              ds=ds,\n                              annexkey=res_addurls['annexkey'],\n                              type=\"file\",\n                              path=filename_abs,\n                              **st_kwargs)\n\n\nclass RegisterUrl(object):\n    \"\"\"Create files (without content) from user-supplied keys and register URLs.\n    \"\"\"\n\n    def __init__(self, ds, repo=None):\n        self.ds = ds\n        self.repo = repo or ds.repo\n        self._err_res = get_status_dict(action=\"addurls\", ds=self.ds,\n                                        type=\"file\", status=\"error\")\n        self.use_pointer = repo.is_managed_branch()\n\n    def examinekey(self, parsed_key, filename, migrate=False):\n        opts = []\n        if migrate:\n            opts.append(\"--migrate-to-backend=\" + parsed_key[\"target_backend\"])\n        opts.extend([\"--filename=\" + filename, parsed_key[\"key\"]])\n        return self.repo.call_annex_records([\"examinekey\"] + opts)[0]\n\n    def fromkey(self, key, filename):\n        return self.repo.call_annex_records(\n            [\"fromkey\", \"--force\", key, filename])[0]\n\n    def registerurl(self, key, url):\n        self.repo.call_annex([\"registerurl\", key, url])\n\n    def _write_pointer(self, row, ek_info):\n        try:\n            fname = Path(row[\"filename_abs\"])\n            fname.parent.mkdir(exist_ok=True, parents=True)\n            fname.write_text(ek_info[\"objectpointer\"])\n        except Exception as exc:\n            message = exc_str(exc)\n            status = \"error\"\n        else:\n            message = \"registered URL\"\n            status = \"ok\"\n        return get_status_dict(action=\"addurls\", ds=self.ds, type=\"file\",\n                               status=status, message=message)\n\n    def __call__(self, row):\n        filename = row[\"ds_filename\"]\n        try:\n            parsed_key = row[\"key\"]\n            migrate = \"target_backend\" in parsed_key\n            use_pointer = self.use_pointer\n            ek_info = None\n            if use_pointer or migrate:\n                ek_info = self.examinekey(parsed_key, filename,\n                                          migrate=migrate)\n                if not ek_info:\n                    yield dict(self._err_res,\n                               path=row[\"filename_abs\"],\n                               message=(\"Failed to get information for %s\",\n                                        parsed_key))\n                    return\n                key = ek_info[\"key\"]\n            else:\n                key = parsed_key[\"key\"]\n\n            self.registerurl(key, row[\"url\"])\n            if use_pointer:\n                res = self._write_pointer(row, ek_info)\n            else:\n                res = annexjson2result(self.fromkey(key, filename),\n                                       self.ds, type=\"file\", logger=lgr)\n                if not res.get(\"message\"):\n                    res[\"message\"] = \"registered URL\"\n        except CommandError as exc:\n            yield dict(self._err_res,\n                       path=row[\"filename_abs\"],\n                       message=exc_str(exc))\n        else:\n            yield res\n\n\n# Note: If any other modules end up needing these batch operations, this should\n# find a new home.\n\n\nclass BatchedRegisterUrl(RegisterUrl):\n    \"\"\"Like `RegisterUrl`, but use batched commands underneath.\n    \"\"\"\n\n    def __init__(self, ds, repo=None):\n        super().__init__(ds, repo)\n        self._batch_commands = {}\n\n    def _batch(self, command, batch_input,\n               output_proc=None, json=False, batch_options=None):\n        bcmd = self._batch_commands.get(command)\n        if not bcmd:\n            repo = self.repo\n            bcmd = repo._batched.get(\n                command, path=repo.path, json=json, output_proc=output_proc,\n                annex_options=batch_options)\n            self._batch_commands[command] = bcmd\n        return bcmd(batch_input)\n\n    def examinekey(self, parsed_key, filename, migrate=False):\n        if migrate:\n            opts = [\"--migrate-to-backend=\" + parsed_key[\"target_backend\"]]\n        else:\n            opts = None\n        return self._batch(\"examinekey\", (parsed_key[\"key\"], filename),\n                           json=True, batch_options=opts)\n\n    def fromkey(self, key, filename):\n        return self._batch(\"fromkey\", (key, filename), json=True,\n                           # --force is needed because the key (usually) does\n                           # not exist in the local repository.\n                           batch_options=[\"--force\"])\n\n    @staticmethod\n    def _ignore(_stdout):\n        return\n\n    def registerurl(self, key, url):\n        self._batch(\"registerurl\", (key, url),\n                    output_proc=self._ignore, json=False)\n\n\ndef _log_filter_addurls(res):\n    return res.get('type') == 'file' and res.get('action') in [\"addurl\", \"addurls\"]\n\n\n@with_result_progress(\"Adding URLs\", log_filter=_log_filter_addurls)\ndef _add_urls(rows, ds, repo, ifexists=None, options=None,\n              drop_after=False, by_key=False):\n    \"\"\"Call `git annex addurl` using information in `rows`.\n    \"\"\"\n    add_url = partial(_add_url, ds=ds, repo=repo,\n                      drop_after=drop_after, options=options)\n    if by_key:\n        # The by_key parameter isn't strictly needed, but it lets us avoid some\n        # setup if --key wasn't specified.\n        if repo.fake_dates_enabled:\n            register_url = RegisterUrl(ds, repo)\n        else:\n            register_url = BatchedRegisterUrl(ds, repo)\n    else:\n        def register_url(*args, **kwargs):\n            raise RuntimeError(\"bug: this should be impossible\")\n\n    add_metadata = {}\n    for row in rows:\n        filename_abs = row[\"filename_abs\"]\n        filename = row[\"ds_filename\"]\n        lgr.debug(\"Adding URLs to %s in %s\", filename, ds.path)\n\n        if os.path.exists(filename_abs) or os.path.islink(filename_abs):\n            if ifexists == \"skip\":\n                yield get_status_dict(action=\"addurls\",\n                                      ds=ds,\n                                      type=\"file\",\n                                      path=filename_abs,\n                                      status=\"notneeded\")\n                continue\n            elif ifexists == \"overwrite\":\n                lgr.debug(\"Removing %s\", filename_abs)\n                unlink(filename_abs)\n            else:\n                lgr.debug(\"File %s already exists\", filename_abs)\n\n        fn = register_url if row.get(\"key\") else add_url\n        all_ok = True\n        for res in fn(row):\n            if res[\"status\"] != \"ok\":\n                all_ok = False\n            yield res\n        if not all_ok:\n            continue\n\n        if row.get(\"meta_args\"):\n            add_metadata[filename] = row[\"meta_args\"]\n\n    if not add_metadata:\n        return\n\n    # For some reason interleaving batched addurl and regular metadata calls\n    # causes multiple commits, so we do need to have it separate.\n    # TODO: figure out with Joey either could be avoided\n    from unittest.mock import patch\n    with patch.object(repo, \"always_commit\", False):\n        for filename, meta in add_metadata.items():\n            lgr.debug(\"Adding metadata to %s in %s\", filename, repo.path)\n            assert not repo.always_commit\n            for a in repo.set_metadata_(filename, add=meta):\n                res = annexjson2result(a, ds, type=\"file\", logger=lgr)\n                if res[\"status\"] == \"ok\":\n                    # Don't show all added metadata for the file because that\n                    # could quickly flood the output.\n                    del res[\"message\"]\n                yield res\n\n\n@build_doc\nclass Addurls(Interface):\n    \"\"\"Create and update a dataset from a list of URLs.\n\n    *Format specification*\n\n    Several arguments take format strings.  These are similar to normal Python\n    format strings where the names from `URL-FILE` (column names for a comma-\n    or tab-separated file or properties for JSON) are available as\n    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can\n    also be used (i.e., \"{0}\" for the first column). Note that a placeholder\n    cannot contain a ':' or '!'.\n\n    In addition, the `FILENAME-FORMAT` arguments has a few special\n    placeholders.\n\n      - _repindex\n\n        The constructed file names must be unique across all fields rows.  To\n        avoid collisions, the special placeholder \"_repindex\" can be added to\n        the formatter.  Its value will start at 0 and increment every time a\n        file name repeats.\n\n      - _url_hostname, _urlN, _url_basename*\n\n        Various parts of the formatted URL are available.  Take\n        \"http://datalad.org/asciicast/seamless_nested_repos.sh\" as an example.\n\n        \"datalad.org\" is stored as \"_url_hostname\".  Components of the URL's\n        path can be referenced as \"_urlN\".  \"_url0\" and \"_url1\" would map to\n        \"asciicast\" and \"seamless_nested_repos.sh\", respectively.  The final\n        part of the path is also available as \"_url_basename\".\n\n        This name is broken down further.  \"_url_basename_root\" and\n        \"_url_basename_ext\" provide access to the root name and extension.\n        These values are similar to the result of os.path.splitext, but, in the\n        case of multiple periods, the extension is identified using the same\n        length heuristic that git-annex uses.  As a result, the extension of\n        \"file.tar.gz\" would be \".tar.gz\", not \".gz\".  In addition, the fields\n        \"_url_basename_root_py\" and \"_url_basename_ext_py\" provide access to\n        the result of os.path.splitext.\n\n      - _url_filename*\n\n        These are similar to _url_basename* fields, but they are obtained with\n        a server request.  This is useful if the file name is set in the\n        Content-Disposition header.\n\n\n    *Examples*\n\n    Consider a file \"avatars.csv\" that contains::\n\n        who,ext,link\n        neurodebian,png,https://avatars3.githubusercontent.com/u/260793\n        datalad,png,https://avatars1.githubusercontent.com/u/8927200\n\n    To download each link into a file name composed of the 'who' and 'ext'\n    fields, we could run::\n\n      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'\n\n    The `-d avatar_ds` is used to create a new dataset in \"$PWD/avatar_ds\".\n\n    If we were already in a dataset and wanted to create a new subdataset in an\n    \"avatars\" subdirectory, we could use \"//\" in the `FILENAME-FORMAT`\n    argument::\n\n      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'\n\n    If the information is represented as JSON lines instead of comma separated\n    values or a JSON array, you can use a utility like jq to transform the JSON\n    lines into an array that addurls accepts::\n\n      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'\n\n    .. note::\n\n       For users familiar with 'git annex addurl': A large part of this\n       plugin's functionality can be viewed as transforming data from\n       `URL-FILE` into a \"url filename\" format that fed to 'git annex addurl\n       --batch --with-files'.\n    \"\"\"\n\n    from datalad.distribution.dataset import datasetmethod\n    from datalad.interface.utils import eval_results\n    from datalad.distribution.dataset import EnsureDataset\n    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr\n    from datalad.support.param import Parameter\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"Add the URLs to this dataset (or possibly subdatasets of\n            this dataset).  An empty or non-existent directory is passed to\n            create a new dataset.  New subdatasets can be specified with\n            `FILENAME-FORMAT`.\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        urlfile=Parameter(\n            args=(\"urlfile\",),\n            metavar=\"URL-FILE\",\n            doc=\"\"\"A file that contains URLs or information that can be used to\n            construct URLs.  Depending on the value of --input-type, this\n            should be a comma- or tab-separated file (with a header as the\n            first row) or a JSON file (structured as a list of objects with\n            string values). If '-', read from standard input, taking the\n            content as JSON when --input-type is at its default value of\n            'ext'.\"\"\"),\n        urlformat=Parameter(\n            args=(\"urlformat\",),\n            metavar=\"URL-FORMAT\",\n            doc=\"\"\"A format string that specifies the URL for each entry.  See\n            the 'Format Specification' section above.\"\"\"),\n        filenameformat=Parameter(\n            args=(\"filenameformat\",),\n            metavar=\"FILENAME-FORMAT\",\n            doc=\"\"\"Like `URL-FORMAT`, but this format string specifies the file\n            to which the URL's content will be downloaded. The name should be a\n            relative path and will be taken as relative to the top-level\n            dataset, regardless of whether it is specified via [PY: `dataset`\n            PY][CMD: --dataset CMD] or inferred. The file name may contain\n            directories. The separator \"//\" can be used to indicate that the\n            left-side directory should be created as a new subdataset. See the\n            'Format Specification' section above.\"\"\"),\n        input_type=Parameter(\n            args=(\"-t\", \"--input-type\"),\n            metavar=\"TYPE\",\n            doc=\"\"\"Whether `URL-FILE` should be considered a CSV file, TSV\n            file, or JSON file. The default value, \"ext\", means to consider\n            `URL-FILE` as a JSON file if it ends with \".json\" or a TSV file if\n            it ends with \".tsv\". Otherwise, treat it as a CSV file.\"\"\",\n            constraints=EnsureChoice(*INPUT_TYPES)),\n        exclude_autometa=Parameter(\n            args=(\"-x\", \"--exclude_autometa\"),\n            metavar=\"REGEXP\",\n            doc=\"\"\"By default, metadata field=value pairs are constructed with\n            each column in `URL-FILE`, excluding any single column that is\n            specified via `URL-FORMAT`.  This argument can be used to exclude\n            columns that match a regular expression.  If set to '*' or an empty\n            string, automatic metadata extraction is disabled completely.  This\n            argument does not affect metadata set explicitly with --meta.\"\"\"),\n        meta=Parameter(\n            args=(\"-m\", \"--meta\",),\n            metavar=\"FORMAT\",\n            action=\"append\",\n            doc=\"\"\"A format string that specifies metadata.  It should be\n            structured as \"<field>=<value>\".  As an example, \"location={3}\"\n            would mean that the value for the \"location\" metadata field should\n            be set the value of the fourth column.  This option can be given\n            multiple times.\"\"\"),\n        key=Parameter(\n            args=(\"--key\",),\n            metavar=\"FORMAT\",\n            doc=\"\"\"A format string that specifies an annex key for the file\n            content. In this case, the file is not downloaded; instead the key\n            is used to create the file without content. The value should be\n            structured as \"[et:]<input backend>[-s<bytes>]--<hash>\". The\n            optional \"et:\" prefix, which requires git-annex 8.20201116 or\n            later, signals to toggle extension state of the input backend\n            (i.e., MD5 vs MD5E). As an example, \"et:MD5-s{size}--{md5sum}\"\n            would use the 'md5sum' and 'size' columns to construct the key,\n            migrating the key from MD5 to MD5E, with an extension based on the\n            file name. Note: If the *input* backend itself is an annex\n            extension backend (i.e., a backend with a trailing \"E\"), the key's\n            extension will not be updated to match the extension of the\n            corresponding file name. Thus, unless the input keys and file names\n            are generated from git-annex, it is recommended to avoid using\n            extension backends as input. If an extension is desired, use the\n            plain variant as input and prepend \"et:\" so that git-annex will\n            migrate from the plain backend to the extension variant.\"\"\"),\n        message=Parameter(\n            args=(\"--message\",),\n            metavar=\"MESSAGE\",\n            doc=\"\"\"Use this message when committing the URL additions.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        dry_run=Parameter(\n            args=(\"-n\", \"--dry-run\"),\n            action=\"store_true\",\n            doc=\"\"\"Report which URLs would be downloaded to which files and\n            then exit.\"\"\"),\n        fast=Parameter(\n            args=(\"--fast\",),\n            action=\"store_true\",\n            doc=\"\"\"If True, add the URLs, but don't download their content.\n            Underneath, this passes the --fast flag to `git annex addurl`.\"\"\"),\n        ifexists=Parameter(\n            args=(\"--ifexists\",),\n            doc=\"\"\"What to do if a constructed file name already exists.  The\n            default behavior is to proceed with the `git annex addurl`, which\n            will fail if the file size has changed.  If set to 'overwrite',\n            remove the old file before adding the new one.  If set to 'skip',\n            do not add the new file.\"\"\",\n            constraints=EnsureChoice(None, \"overwrite\", \"skip\")),\n        missing_value=Parameter(\n            args=(\"--missing-value\",),\n            metavar=\"VALUE\",\n            doc=\"\"\"When an empty string is encountered, use this value\n            instead.\"\"\",\n            constraints=EnsureNone() | EnsureStr()),\n        save=nosave_opt,\n        version_urls=Parameter(\n            args=(\"--version-urls\",),\n            action=\"store_true\",\n            doc=\"\"\"Try to add a version ID to the URL. This currently only has\n            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is\n            not yet supported, but any URL that already contains a \"versionId=\"\n            parameter will be used as is.\"\"\"),\n        cfg_proc=Parameter(\n            args=(\"-c\", \"--cfg-proc\"),\n            metavar=\"PROC\",\n            action='append',\n            doc=\"\"\"Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when\n            calling `create` to make datasets.\"\"\"),\n        jobs=jobs_opt,\n        drop_after=Parameter(\n            args=(\"--drop-after\",),\n            action=\"store_true\",\n            doc=\"\"\"drop files after adding to annex\"\"\",\n        ),\n    )\n\n    @staticmethod\n    @datasetmethod(name='addurls')\n    @eval_results\n    def __call__(dataset, urlfile, urlformat, filenameformat,\n                 input_type=\"ext\", exclude_autometa=None, meta=None, key=None,\n                 message=None, dry_run=False, fast=False, ifexists=None,\n                 missing_value=None, save=True, version_urls=False,\n                 cfg_proc=None, jobs=None, drop_after=False):\n        # Temporarily work around gh-2269.\n        url_file = urlfile\n        url_format, filename_format = urlformat, filenameformat\n\n        from requests.exceptions import RequestException\n\n        from datalad.distribution.dataset import Dataset, require_dataset\n        from datalad.support.annexrepo import AnnexRepo\n\n        lgr = logging.getLogger(\"datalad.plugin.addurls\")\n\n        ds = require_dataset(dataset, check_installed=False)\n        repo = ds.repo\n        st_dict = get_status_dict(action=\"addurls\", ds=ds)\n        if repo and not isinstance(repo, AnnexRepo):\n            yield dict(st_dict, status=\"error\", message=\"not an annex repo\")\n            return\n\n        if key:\n            old_examinekey = external_versions[\"cmd:annex\"] < \"8.20201116\"\n            if old_examinekey:\n                old_msg = None\n                if key.startswith(\"et:\"):\n                    old_msg = (\"et: prefix of `key` option requires \"\n                               \"git-annex 8.20201116 or later\")\n                elif repo.is_managed_branch():\n                    old_msg = (\"Using `key` option on adjusted branch \"\n                               \"requires git-annex 8.20201116 or later\")\n                if old_msg:\n                    yield dict(st_dict, status=\"error\", message=old_msg)\n                    return\n\n        if url_file != \"-\":\n            url_file = str(resolve_path(url_file, dataset))\n\n        if input_type == \"ext\":\n            if url_file == \"-\":\n                input_type = \"json\"\n            else:\n                extension = os.path.splitext(url_file)[1]\n                if extension == \".json\":\n                    input_type = \"json\"\n                elif extension == \".tsv\":\n                    input_type = \"tsv\"\n                else:\n                    input_type = \"csv\"\n\n        fd = sys.stdin if url_file == \"-\" else open(url_file)\n        try:\n            try:\n                rows, subpaths = extract(fd, input_type,\n                                         url_format, filename_format,\n                                         exclude_autometa, meta, key,\n                                         dry_run,\n                                         missing_value)\n            except (ValueError, RequestException) as exc:\n                yield dict(st_dict, status=\"error\", message=exc_str(exc))\n                return\n        finally:\n            if fd is not sys.stdin:\n                fd.close()\n\n        if not rows:\n            yield dict(st_dict, status=\"notneeded\",\n                       message=\"No rows to process\")\n            return\n\n        if len(rows) != len(set(row[\"filename\"] for row in rows)):\n            yield dict(st_dict, status=\"error\",\n                       message=(\"There are file name collisions; \"\n                                \"consider using {_repindex}\"))\n            return\n\n        if dry_run:\n            for subpath in subpaths:\n                lgr.info(\"Would create a subdataset at %s\", subpath)\n            for row in rows:\n                lgr.info(\"Would %s %s to %s\",\n                         \"register\" if row.get(\"key\") else \"download\",\n                         row[\"url\"],\n                         os.path.join(ds.path, row[\"filename\"]))\n                if \"meta_args\" in row:\n                    lgr.info(\"Metadata: %s\",\n                             sorted(u\"{}={}\".format(k, v)\n                                    for k, v in row[\"meta_args\"].items()))\n            yield dict(st_dict, status=\"ok\", message=\"dry-run finished\")\n            return\n\n        if not repo:\n            # Populate a new dataset with the URLs.\n            yield from ds.create(\n                result_xfm=None,\n                return_type='generator',\n                cfg_proc=cfg_proc)\n\n        annex_options = [\"--fast\"] if fast else []\n\n        # to be populated by addurls_to_ds\n        files_to_add = set()\n        created_subds = []\n\n        def addurls_to_ds(args):\n            \"\"\"The \"consumer\" for ProducerConsumer parallel execution\"\"\"\n            subpath, rows = args\n\n            ds_path = ds.path  # shortcut on closure from outside\n\n            # technically speaking, subds might be the ds\n            if subpath:\n                subds_path = os.path.join(ds_path, subpath)\n            else:\n                subds_path = ds_path\n\n            for row in rows:\n                # Add additional information that we'll need for various\n                # operations.\n                filename_abs = op.join(ds_path, row[\"filename\"])\n                ds_filename = op.relpath(filename_abs, subds_path)\n                row.update({\"filename_abs\": filename_abs,\n                            \"ds_filename\": ds_filename})\n\n            subds = Dataset(subds_path)\n\n            if subds.is_installed():\n                lgr.debug(\n                    \"Not creating subdataset at existing path: %s\",\n                    subds_path)\n            else:\n                yield from subds.create(result_xfm=None,\n                                        cfg_proc=cfg_proc,\n                                        return_type='generator')\n                created_subds.append(subpath)\n            repo = subds.repo  # \"expensive\" so we get it once\n\n            if version_urls:\n                num_urls = len(rows)\n                log_progress(lgr.info, \"addurls_versionurls\",\n                             \"Versioning %d URLs\", num_urls,\n                             label=\"Versioning URLs\",\n                             total=num_urls, unit=\" URLs\")\n                for row in rows:\n                    url = row[\"url\"]\n                    try:\n                        # TODO: make get_versioned_url more efficient while going\n                        # through the same bucket(s)\n                        row[\"url\"] = get_versioned_url(url)\n                    except (ValueError, NotImplementedError) as exc:\n                        # We don't expect this to happen because get_versioned_url\n                        # should return the original URL if it isn't an S3 bucket.\n                        # It only raises exceptions if it doesn't know how to\n                        # handle the scheme for what looks like an S3 bucket.\n                        lgr.warning(\"error getting version of %s: %s\",\n                                    row[\"url\"], exc_str(exc))\n                    log_progress(lgr.info, \"addurls_versionurls\",\n                                 \"Versioned result for %s: %s\", url, row[\"url\"],\n                                 update=1, increment=True)\n                log_progress(lgr.info, \"addurls_versionurls\", \"Finished versioning URLs\")\n\n            subds_files_to_add = set()\n            for r in _add_urls(rows, subds, repo,\n                               ifexists=ifexists, options=annex_options,\n                               drop_after=drop_after, by_key=key):\n                if r[\"status\"] == \"ok\":\n                    subds_files_to_add.add(r[\"path\"])\n                yield r\n\n            files_to_add.update(subds_files_to_add)\n            pass  # end of addurls_to_ds\n\n\n        # We need to group by dataset since otherwise we will initiate\n        # batched annex process per each subdataset, which might be infeasible\n        # in any use-case with a considerable number of subdatasets.\n        # Also groupping allows us for parallelization across datasets, and avoids\n        # proliferation of commit messages upon creation of each individual subdataset.\n\n        def keyfn(d):\n            # The top-level dataset has a subpath of None.\n            return d.get(\"subpath\") or \"\"\n\n        # We need to serialize itertools.groupby .\n        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]\n\n        # There could be \"intermediate\" subdatasets which have no rows but would need\n        # their datasets created and saved, so let's add them\n        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)\n        nrows_by_ds_orig = len(rows_by_ds)\n        if add_subpaths:\n            for subpath in add_subpaths:\n                rows_by_ds.append((subpath, tuple()))\n            # and now resort them again since we added\n            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])\n\n        # We want to provide progress overall files not just datasets\n        # so our total will be just a len of rows\n        def agg_files(*args, **kwargs):\n            return len(rows)\n\n        yield from ProducerConsumerProgressLog(\n            rows_by_ds,\n            addurls_to_ds,\n            agg=agg_files,\n            # It is ok to start with subdatasets since top dataset already exists\n            safe_to_consume=partial(no_parentds_in_futures, skip=(\"\", None, \".\")),\n            # our producer provides not only dataset paths and also rows, take just path\n            producer_future_key=lambda row_by_ds: row_by_ds[0],\n            jobs=jobs,\n            # Logging options\n            # we will be yielding all kinds of records, but of interest for progress\n            # reporting only addurls on files\n            # note: annexjson2result overrides 'action' with 'command' content from\n            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.\n            # TODO: see if it is all ok, since we might be now yielding both\n            # addurls and addurl records.\n            log_filter=_log_filter_addurls,\n            unit=\"files\",\n            lgr=lgr,\n        )\n\n        if save:\n            extra_msgs = []\n            if created_subds:\n                extra_msgs.append(f\"{len(created_subds)} subdatasets were created\")\n            if extra_msgs:\n                extra_msgs.append('')\n            message_addurls = message or f\"\"\"\\\n[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs\n\n{os.linesep.join(extra_msgs)}\nurl_file='{urlfile}'\nurl_format='{url_format}'\nfilename_format='{filenameformat}'\"\"\"\n\n            # save all in bulk\n            files_to_add.update([r[0] for r in rows_by_ds])\n            yield from ds.save(\n                list(files_to_add),\n                message=message_addurls,\n                jobs=jobs,\n                return_type='generator')\n\n\n__datalad_plugin__ = Addurls\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Extract Function Fails in Tests Due to Missing `input_type` Parameter

**Description:**
After recent changes to the `extract` function to support passing a list of records, several tests are failing because they still call `extract` without the required `input_type` argument.

**Example Buggy Code:**
```python
info, subpaths = au.extract(
    ST_DATA['rows'],
    url_format='{name}_{debut_season}.com',
    filename_format='{age_group}//{now_dead}//{name}.csv'
)
```

**Expected Behavior:**
The `extract` function should process the provided records correctly without requiring the `input_type` parameter, accommodating the new functionality of accepting a list of records.

**Actual Behavior:**
Calling `extract` without the `input_type` parameter results in the following error:
```
TypeError: extract() missing 1 required positional argument: 'input_type'
```
This prevents the tests from executing successfully and indicates a mismatch between the function's updated signature and its usage in the tests.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/plugin/addurls.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Create and update a dataset from a list of URLs.
"""

from collections.abc import Mapping

import logging
import os
import re
import string
import sys

from functools import partial
from urllib.parse import urlparse

from datalad.support.external_versions import external_versions
import datalad.support.path as op
from datalad.distribution.dataset import resolve_path
from datalad.dochelpers import exc_str
from datalad.log import log_progress, with_result_progress
from datalad.interface.base import Interface
from datalad.interface.base import build_doc
from datalad.interface.results import annexjson2result, get_status_dict
from datalad.interface.common_opts import (
    jobs_opt,
    nosave_opt,
)
from datalad.support.exceptions import CommandError
from datalad.support.itertools import groupby_sorted
from datalad.support.network import get_url_filename
from datalad.support.path import split_ext
from datalad.support.parallel import (
    ProducerConsumerProgressLog,
    no_parentds_in_futures,
)
from datalad.support.s3 import get_versioned_url
from datalad.utils import (
    ensure_list,
    get_suggestions_msg,
    Path,
    unlink,
)

lgr = logging.getLogger("datalad.plugin.addurls")

__docformat__ = "restructuredtext"


class Formatter(string.Formatter):
    """Formatter that gives precedence to custom keys.

    The first positional argument to the `format` call should be a
    mapping whose keys are exposed as placeholders (e.g.,
    "{key1}.py").

    Parameters
    ----------
    idx_to_name : dict
        A mapping from a positional index to a key.  If not provided,
        "{N}" elements are not supported.
    missing : str, optional
        When column lookup results in an empty string, use this value in
        its place.
    """

    def __init__(self, idx_to_name=None, missing_value=None):
        self.idx_to_name = idx_to_name or {}
        self.missing = missing_value

    def format(self, format_string, *args, **kwargs):
        if not isinstance(args[0], Mapping):
            raise ValueError("First positional argument should be mapping")
        return super(Formatter, self).format(format_string, *args, **kwargs)

    def get_value(self, key, args, kwargs):
        """Look for key's value in `args[0]` mapping first.
        """
        # FIXME: This approach will fail for keys that contain "!" and
        # ":" because they'll be interpreted as formatting flags.
        data = args[0]

        name = key
        try:
            key_int = int(key)
        except ValueError:
            pass
        else:
            name = self.idx_to_name[key_int]

        try:
            value = data[name]
        except KeyError:
            return super(Formatter, self).get_value(
                key, args, kwargs)

        if self.missing is not None and isinstance(value, str):
            return value or self.missing
        return value

    def convert_field(self, value, conversion):
        if conversion == 'l':
            return str(value).lower()
        return super(Formatter, self).convert_field(value, conversion)


class RepFormatter(Formatter):
    """Extend Formatter to support a {_repindex} placeholder.
    """

    def __init__(self, *args, **kwargs):
        super(RepFormatter, self).__init__(*args, **kwargs)
        self.repeats = {}
        self.repindex = 0

    def format(self, *args, **kwargs):
        self.repindex = 0
        result = super(RepFormatter, self).format(*args, **kwargs)
        if result in self.repeats:
            self.repindex = self.repeats[result] + 1
            self.repeats[result] = self.repindex
            result = super(RepFormatter, self).format(*args, **kwargs)
        else:
            self.repeats[result] = 0
        return result

    def get_value(self, key, args, kwargs):
        args[0]["_repindex"] = self.repindex
        return super(RepFormatter, self).get_value(key, args, kwargs)


def clean_meta_args(args):
    """Process metadata arguments.

    Parameters
    ----------
    args : iterable of str
        Formatted metadata arguments for 'git-annex metadata --set'.

    Returns
    -------
    A dict mapping field names to values.
    """
    results = {}
    for arg in args:
        parts = [x.strip() for x in arg.split("=", 1)]
        if len(parts) == 2:
            if not parts[0]:
                raise ValueError("Empty field name")
            field, value = parts
        else:
            raise ValueError("meta argument isn't in 'field=value' format")

        if not value:
            # The `url_file` may have an empty value.
            continue
        results[field] = value
    return results


def get_subpaths(filename):
    """Convert "//" marker in `filename` to a list of subpaths.

    >>> from datalad.plugin.addurls import get_subpaths
    >>> get_subpaths("p1/p2//p3/p4//file")
    ('p1/p2/p3/p4/file', ['p1/p2', 'p1/p2/p3/p4'])

    Note: With Python 3, the subpaths could be generated with

        itertools.accumulate(filename.split("//")[:-1], os.path.join)

    Parameters
    ----------
    filename : str
        File name with "//" marking subpaths.

    Returns
    -------
    A tuple of the filename with any "//" collapsed to a single
    separator and a list of subpaths (str).
    """
    if "//" not in filename:
        return filename, []

    spaths = []
    for part in filename.split("//")[:-1]:
        path = os.path.join(spaths[-1], part) if spaths else part
        spaths.append(path)
    return filename.replace("//", os.path.sep), spaths


def is_legal_metafield(name):
    """Test whether `name` is a valid metadata field.

    The set of permitted characters is taken from git-annex's
    MetaData.hs:legalField.
    """
    return bool(re.match(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z", name))


def filter_legal_metafield(fields):
    """Remove illegal names from `fields`.

    Note: This is like `filter(is_legal_metafield, fields)` but the
    dropped values are logged.
    """
    legal = []
    for field in fields:
        if is_legal_metafield(field):
            legal.append(field)
        else:
            lgr.debug("%s is not a valid metadata field name; dropping",
                      field)
    return legal


class AnnexKeyParser(object):
    """Parse a full annex key into subparts.

    The key may have an "et:" prefix appended, which signals that the backend's
    extension state should be toggled.

    See <https://git-annex.branchable.com/internals/key_format/>.

    Parameters
    ----------
    format_fn : callable
        Function that takes a format string and a row and returns the full key.
    format_string : str
        Format string for the full key.
    """

    def __init__(self, format_fn, format_string):
        self.format_fn = format_fn
        self.format_string = format_string
        self.regexp = re.compile(r"(?P<et>et:)?"
                                 r"(?P<backend>[A-Z0-9]+)"
                                 r"(?:-[^-]+)?"
                                 r"--(?P<keyname>[^/\n]+)\Z")
        self.empty = "".join(i[0]
                             for i in string.Formatter().parse(format_string))

    @staticmethod
    def _validate_common(backend, key):
        if backend.endswith("E"):
            ext = True
            backend = backend[:-1]
        else:
            ext = False

        expected_lengths = {"MD5": 32,
                            "SHA1": 40,
                            "SHA256": 64}

        expected_len = expected_lengths.get(backend)
        if not expected_len:
            return

        if ext and "." in key:
            key = key.split(".", maxsplit=1)[0]

        if len(key) != expected_len:
            raise ValueError("{} key does not have expected length of {}: {}"
                             .format(backend, expected_len, key))

        if not re.match(r"[a-f0-9]+\Z", key):
            raise ValueError("{} key has non-hexadecimal character: {}"
                             .format(backend, key))

    def parse(self, row):
        """Format the key with the fields in `row` and parse it.

        Returns
        -------
        A dictionary with the following keys that match their counterparts in
        the output of `git annex examinekey --json`: "key" (the full annex
        key), "backend", and "keyname". If the key had an "et:" prefix, there
        is also a "target_backend" key.

        Raises
        ------
        ValueError if the formatted value doesn't look like a valid key
        """
        try:
            key = self.format_fn(self.format_string, row)
        except KeyError as exc:
            lgr.debug("Row missing fields for --key: %s",
                      exc_str(exc))
            return {}

        if key == self.empty:
            lgr.debug("All fields in --key's value are empty in row: %s", row)
            # We got the same string that'd we get if all the fields were
            # empty, so this doesn't have a key.
            return {}

        match = self.regexp.match(key)
        if match:
            info = match.groupdict()
            et = info.pop("et", None)
            if et:
                info["key"] = key[3:]  # Drop "et:" from full key.
                backend = info["backend"]
                if backend.endswith("E"):
                    info["target_backend"] = backend[:-1]
                else:
                    info["target_backend"] = backend + "E"
            else:
                info["key"] = key
            self._validate_common(info["backend"], info["keyname"])
            return info
        else:
            raise ValueError(
                "Key does not match expected "
                "[et:]<backend>[-[CmsS]NNN]--<key> format: {}"
                .format(key))


def get_fmt_names(format_string):
    """Yield field names in `format_string`.
    """
    for _, name, _, _ in string.Formatter().parse(format_string):
        if name:
            yield name


def fmt_to_name(format_string, num_to_name):
    """Try to map a format string to a single name.

    Parameters
    ----------
    format_string : string
    num_to_name : dict
        A dictionary that maps from an integer to a column name.  This
        enables mapping the format string to an integer to a name.

    Returns
    -------
    A placeholder name if `format_string` consists of a single
    placeholder and no other text.  Otherwise, None is returned.
    """
    parsed = list(string.Formatter().parse(format_string))
    if len(parsed) != 1:
        # It's an empty string or there's more than one placeholder.
        return
    if parsed[0][0]:
        # Format string contains text before the placeholder.
        return

    name = parsed[0][1]
    if not name:
        # The field name is empty.
        return

    try:
        return num_to_name[int(name)]
    except (KeyError, ValueError):
        return name


INPUT_TYPES = ["ext", "csv", "tsv", "json"]


def _read(stream, input_type):
    if input_type in ["csv", "tsv"]:
        import csv
        csvrows = csv.reader(stream,
                             delimiter="\t" if input_type == "tsv" else ",")
        try:
            headers = next(csvrows)
        except StopIteration:
            raise ValueError("Failed to read {} rows from {}"
                             .format(input_type.upper(), stream))
        lgr.debug("Taking %s fields from first line as headers: %s",
                  len(headers), headers)
        idx_map = dict(enumerate(headers))
        rows = [dict(zip(headers, r)) for r in csvrows]
    elif input_type == "json":
        import json
        try:
            rows = json.load(stream)
        except json.decoder.JSONDecodeError as e:
            raise ValueError(
                "Failed to read JSON from stream {}: {}"
                .format(stream, exc_str(e)))
        # For json input, we do not support indexing by position,
        # only names.
        idx_map = {}
    else:
        raise ValueError(
            "input_type {} is invalid. Known values: {}"
            .format(input_type, ", ".join(INPUT_TYPES)))
    return rows, idx_map


def _get_placeholder_exception(exc, msg_prefix, known):
    """Recast KeyError as a ValueError with close-match suggestions.
    """
    value = exc.args[0]
    if isinstance(value, str):
        sugmsg = get_suggestions_msg(value, known)
    else:
        sugmsg = "Out-of-bounds or unsupported index."
    # Note: Keeping this a KeyError is probably more appropriate but then the
    # entire message, which KeyError takes as the key, will be rendered with
    # outer quotes.
    return ValueError("{}: {}{}{}"
                      .format(msg_prefix, exc, ". " if sugmsg else "", sugmsg))


def _format_filenames(format_fn, rows, row_infos):
    subpaths = set()
    for row, info in zip(rows, row_infos):
        try:
            filename = format_fn(row)
        except KeyError as exc:
            raise _get_placeholder_exception(
                exc, "Unknown placeholder in file name", row)
        filename, spaths = get_subpaths(filename)
        subpaths |= set(spaths)
        info["filename"] = filename
        info["subpath"] = spaths[-1] if spaths else None
    return subpaths


def get_file_parts(filename, prefix="name"):
    """Assign a name to various parts of a file.

    Parameters
    ----------
    filename : str
        A file name (no leading path is permitted).
    prefix : str
        Prefix to prepend to the key names.

    Returns
    -------
    A dict mapping each part to a value.
    """
    root, ext = split_ext(filename)
    root_py, ext_py = os.path.splitext(filename)

    return {prefix: filename,
            prefix + "_root": root,
            prefix + "_ext": ext,
            prefix + "_root_py": root_py,
            prefix + "_ext_py": ext_py}


def get_url_parts(url):
    """Assign a name to various parts of the URL.

    Parameters
    ----------
    url : str

    Returns
    -------
    A dict with keys `_url_hostname` and, for a path with N+1 parts,
    '_url0' through '_urlN' .  There is also a `_url_basename` key for
    the rightmost part of the path.
    """
    parsed = urlparse(url)
    if not parsed.netloc:
        return {}

    names = {"_url_hostname": parsed.netloc}

    path = parsed.path.strip("/")
    if not path:
        return names

    url_parts = path.split("/")
    for pidx, part in enumerate(url_parts):
        names["_url{}".format(pidx)] = part
    basename = url_parts[-1]
    names["_url_basename"] = basename
    names.update(get_file_parts(basename, prefix="_url_basename"))
    return names


def add_extra_filename_values(filename_format, rows, urls, dry_run):
    """Extend `rows` with values for special formatting fields.
    """
    file_fields = list(get_fmt_names(filename_format))
    if any(i.startswith("_url") for i in file_fields):
        for row, url in zip(rows, urls):
            row.update(get_url_parts(url))

    if any(i.startswith("_url_filename") for i in file_fields):
        if dry_run:  # Don't waste time making requests.
            dummy = get_file_parts("BASE.EXT", "_url_filename")
            for idx, row in enumerate(rows):
                row.update(
                    {k: v + str(idx) for k, v in dummy.items()})
        else:
            num_urls = len(urls)
            log_progress(lgr.info, "addurls_requestnames",
                         "Requesting file names for %d URLs", num_urls,
                         label="Requesting names", total=num_urls,
                         unit=" Files")
            for row, url in zip(rows, urls):
                # If we run into any issues here, we're just going to raise an
                # exception and then abort inside dlplugin.  It'd be good to
                # disentangle this from `extract` so that we could yield an
                # individual error, drop the row, and keep going.
                filename = get_url_filename(url)
                if filename:
                    row.update(get_file_parts(filename, "_url_filename"))
                else:
                    raise ValueError(
                        "{} does not contain a filename".format(url))
                log_progress(lgr.info, "addurls_requestnames",
                             "%s returned for %s", url, filename,
                             update=1, increment=True)
            log_progress(lgr.info, "addurls_requestnames",
                         "Finished requesting file names")


def sort_paths(paths):
    """Sort `paths` by directory level and then alphabetically.

    Parameters
    ----------
    paths : iterable of str

    Returns
    -------
    Generator of sorted paths.
    """
    def level_and_name(p):
        return p.count(os.path.sep), p

    yield from sorted(paths, key=level_and_name)


def extract(stream, input_type, url_format="{0}", filename_format="{1}",
            exclude_autometa=None, meta=None, key=None,
            dry_run=False, missing_value=None):
    """Extract and format information from `url_file`.

    Parameters
    ----------
    stream : file object
        Items used to construct the file names and URLs.
    input_type : {'csv', 'tsv', 'json'}

    All other parameters match those described in `AddUrls`.

    Returns
    -------
    A tuple where the first item is a list with a dict of extracted information
    for each row in `stream` and the second item a list subdataset paths,
    sorted breadth-first.
    """
    meta = ensure_list(meta)

    rows, colidx_to_name = _read(stream, input_type)
    if not rows:
        lgr.warning("No rows found in %s", stream)
        return [], []

    # Formatter for everything but file names
    fmt = Formatter(colidx_to_name, missing_value)
    format_url = partial(fmt.format, url_format)

    auto_meta_args = []
    if exclude_autometa not in ["*", ""]:
        urlcol = fmt_to_name(url_format, colidx_to_name)
        # TODO: Try to normalize invalid fields, checking for any
        # collisions.
        metacols = (c for c in sorted(rows[0].keys()) if c != urlcol)
        if exclude_autometa:
            metacols = (c for c in metacols
                        if not re.search(exclude_autometa, c))
        metacols = filter_legal_metafield(metacols)
        auto_meta_args = [c + "=" + "{" + c + "}" for c in metacols]

    # Unlike `filename_format` and `url_format`, `meta` is a list
    # because meta may be given multiple times on the command line.
    formats_meta = [partial(fmt.format, m) for m in meta + auto_meta_args]

    info_fns = []
    if formats_meta:
        def set_meta_args(info, row):
            info["meta_args"] = clean_meta_args(fmt(row)
                                                for fmt in formats_meta)
        info_fns.append(set_meta_args)
    if key:
        key_parser = AnnexKeyParser(fmt.format, key)

        def set_key(info, row):
            info["key"] = key_parser.parse(row)
        info_fns.append(set_key)

    rows_with_url = []
    infos = []
    for row in rows:
        try:
            url = format_url(row)
        except KeyError as exc:
            raise _get_placeholder_exception(
                exc, "Unknown placeholder in URL", row)
        if not url or url == missing_value:
            continue  # pragma: no cover, peephole optimization
        rows_with_url.append(row)
        info = {"url": url}
        for fn in info_fns:
            fn(info, row)
        infos.append(info)

    n_dropped = len(rows) - len(rows_with_url)
    if n_dropped:
        lgr.warning("Dropped %d row(s) that had an empty URL", n_dropped)

    # Format the filename in a second pass so that we can provide
    # information about the formatted URLs.
    add_extra_filename_values(filename_format, rows_with_url,
                              [i["url"] for i in infos],
                              dry_run)

    # For the file name, we allow the _repindex special key.
    format_filename = partial(
        RepFormatter(colidx_to_name, missing_value).format,
        filename_format)
    subpaths = _format_filenames(format_filename, rows_with_url, infos)
    return infos, list(sort_paths(subpaths))


def _add_url(row, ds, repo, options=None, drop_after=False):
    filename_abs = row["filename_abs"]
    filename = row["ds_filename"]
    try:
        out_json = repo.add_url_to_file(filename, row["url"],
                                        batch=True, options=options)
    except CommandError as exc:
        yield get_status_dict(action="addurls",
                              ds=ds,
                              type="file",
                              path=filename_abs,
                              message=exc_str(exc),
                              status="error")
        return

    # In the case of an error, the json object has file=None.
    if out_json["file"] is None:
        out_json["file"] = filename_abs
    res_addurls = annexjson2result(
        out_json, ds, action="addurls",
        type="file", logger=lgr)
    yield res_addurls

    if not res_addurls["status"] == "ok":
        return

    if drop_after and 'annexkey' in res_addurls:
        # unfortunately .drop has no batched mode, and drop_key ATM would
        # raise AssertionError if not success, and otherwise return nothing
        try:
            repo.drop_key(res_addurls['annexkey'], batch=True)
            st_kwargs = dict(status="ok")
        except (AssertionError, CommandError) as exc:
            st_kwargs = dict(message=exc_str(exc),
                             status="error")
        yield get_status_dict(action="drop",
                              ds=ds,
                              annexkey=res_addurls['annexkey'],
                              type="file",
                              path=filename_abs,
                              **st_kwargs)


class RegisterUrl(object):
    """Create files (without content) from user-supplied keys and register URLs.
    """

    def __init__(self, ds, repo=None):
        self.ds = ds
        self.repo = repo or ds.repo
        self._err_res = get_status_dict(action="addurls", ds=self.ds,
                                        type="file", status="error")
        self.use_pointer = repo.is_managed_branch()

    def examinekey(self, parsed_key, filename, migrate=False):
        opts = []
        if migrate:
            opts.append("--migrate-to-backend=" + parsed_key["target_backend"])
        opts.extend(["--filename=" + filename, parsed_key["key"]])
        return self.repo.call_annex_records(["examinekey"] + opts)[0]

    def fromkey(self, key, filename):
        return self.repo.call_annex_records(
            ["fromkey", "--force", key, filename])[0]

    def registerurl(self, key, url):
        self.repo.call_annex(["registerurl", key, url])

    def _write_pointer(self, row, ek_info):
        try:
            fname = Path(row["filename_abs"])
            fname.parent.mkdir(exist_ok=True, parents=True)
            fname.write_text(ek_info["objectpointer"])
        except Exception as exc:
            message = exc_str(exc)
            status = "error"
        else:
            message = "registered URL"
            status = "ok"
        return get_status_dict(action="addurls", ds=self.ds, type="file",
                               status=status, message=message)

    def __call__(self, row):
        filename = row["ds_filename"]
        try:
            parsed_key = row["key"]
            migrate = "target_backend" in parsed_key
            use_pointer = self.use_pointer
            ek_info = None
            if use_pointer or migrate:
                ek_info = self.examinekey(parsed_key, filename,
                                          migrate=migrate)
                if not ek_info:
                    yield dict(self._err_res,
                               path=row["filename_abs"],
                               message=("Failed to get information for %s",
                                        parsed_key))
                    return
                key = ek_info["key"]
            else:
                key = parsed_key["key"]

            self.registerurl(key, row["url"])
            if use_pointer:
                res = self._write_pointer(row, ek_info)
            else:
                res = annexjson2result(self.fromkey(key, filename),
                                       self.ds, type="file", logger=lgr)
                if not res.get("message"):
                    res["message"] = "registered URL"
        except CommandError as exc:
            yield dict(self._err_res,
                       path=row["filename_abs"],
                       message=exc_str(exc))
        else:
            yield res


# Note: If any other modules end up needing these batch operations, this should
# find a new home.


class BatchedRegisterUrl(RegisterUrl):
    """Like `RegisterUrl`, but use batched commands underneath.
    """

    def __init__(self, ds, repo=None):
        super().__init__(ds, repo)
        self._batch_commands = {}

    def _batch(self, command, batch_input,
               output_proc=None, json=False, batch_options=None):
        bcmd = self._batch_commands.get(command)
        if not bcmd:
            repo = self.repo
            bcmd = repo._batched.get(
                command, path=repo.path, json=json, output_proc=output_proc,
                annex_options=batch_options)
            self._batch_commands[command] = bcmd
        return bcmd(batch_input)

    def examinekey(self, parsed_key, filename, migrate=False):
        if migrate:
            opts = ["--migrate-to-backend=" + parsed_key["target_backend"]]
        else:
            opts = None
        return self._batch("examinekey", (parsed_key["key"], filename),
                           json=True, batch_options=opts)

    def fromkey(self, key, filename):
        return self._batch("fromkey", (key, filename), json=True,
                           # --force is needed because the key (usually) does
                           # not exist in the local repository.
                           batch_options=["--force"])

    @staticmethod
    def _ignore(_stdout):
        return

    def registerurl(self, key, url):
        self._batch("registerurl", (key, url),
                    output_proc=self._ignore, json=False)


def _log_filter_addurls(res):
    return res.get('type') == 'file' and res.get('action') in ["addurl", "addurls"]


@with_result_progress("Adding URLs", log_filter=_log_filter_addurls)
def _add_urls(rows, ds, repo, ifexists=None, options=None,
              drop_after=False, by_key=False):
    """Call `git annex addurl` using information in `rows`.
    """
    add_url = partial(_add_url, ds=ds, repo=repo,
                      drop_after=drop_after, options=options)
    if by_key:
        # The by_key parameter isn't strictly needed, but it lets us avoid some
        # setup if --key wasn't specified.
        if repo.fake_dates_enabled:
            register_url = RegisterUrl(ds, repo)
        else:
            register_url = BatchedRegisterUrl(ds, repo)
    else:
        def register_url(*args, **kwargs):
            raise RuntimeError("bug: this should be impossible")

    add_metadata = {}
    for row in rows:
        filename_abs = row["filename_abs"]
        filename = row["ds_filename"]
        lgr.debug("Adding URLs to %s in %s", filename, ds.path)

        if os.path.exists(filename_abs) or os.path.islink(filename_abs):
            if ifexists == "skip":
                yield get_status_dict(action="addurls",
                                      ds=ds,
                                      type="file",
                                      path=filename_abs,
                                      status="notneeded")
                continue
            elif ifexists == "overwrite":
                lgr.debug("Removing %s", filename_abs)
                unlink(filename_abs)
            else:
                lgr.debug("File %s already exists", filename_abs)

        fn = register_url if row.get("key") else add_url
        all_ok = True
        for res in fn(row):
            if res["status"] != "ok":
                all_ok = False
            yield res
        if not all_ok:
            continue

        if row.get("meta_args"):
            add_metadata[filename] = row["meta_args"]

    if not add_metadata:
        return

    # For some reason interleaving batched addurl and regular metadata calls
    # causes multiple commits, so we do need to have it separate.
    # TODO: figure out with Joey either could be avoided
    from unittest.mock import patch
    with patch.object(repo, "always_commit", False):
        for filename, meta in add_metadata.items():
            lgr.debug("Adding metadata to %s in %s", filename, repo.path)
            assert not repo.always_commit
            for a in repo.set_metadata_(filename, add=meta):
                res = annexjson2result(a, ds, type="file", logger=lgr)
                if res["status"] == "ok":
                    # Don't show all added metadata for the file because that
                    # could quickly flood the output.
                    del res["message"]
                yield res


@build_doc
class Addurls(Interface):
    """Create and update a dataset from a list of URLs.

    *Format specification*

    Several arguments take format strings.  These are similar to normal Python
    format strings where the names from `URL-FILE` (column names for a comma-
    or tab-separated file or properties for JSON) are available as
    placeholders. If `URL-FILE` is a CSV or TSV file, a positional index can
    also be used (i.e., "{0}" for the first column). Note that a placeholder
    cannot contain a ':' or '!'.

    In addition, the `FILENAME-FORMAT` arguments has a few special
    placeholders.

      - _repindex

        The constructed file names must be unique across all fields rows.  To
        avoid collisions, the special placeholder "_repindex" can be added to
        the formatter.  Its value will start at 0 and increment every time a
        file name repeats.

      - _url_hostname, _urlN, _url_basename*

        Various parts of the formatted URL are available.  Take
        "http://datalad.org/asciicast/seamless_nested_repos.sh" as an example.

        "datalad.org" is stored as "_url_hostname".  Components of the URL's
        path can be referenced as "_urlN".  "_url0" and "_url1" would map to
        "asciicast" and "seamless_nested_repos.sh", respectively.  The final
        part of the path is also available as "_url_basename".

        This name is broken down further.  "_url_basename_root" and
        "_url_basename_ext" provide access to the root name and extension.
        These values are similar to the result of os.path.splitext, but, in the
        case of multiple periods, the extension is identified using the same
        length heuristic that git-annex uses.  As a result, the extension of
        "file.tar.gz" would be ".tar.gz", not ".gz".  In addition, the fields
        "_url_basename_root_py" and "_url_basename_ext_py" provide access to
        the result of os.path.splitext.

      - _url_filename*

        These are similar to _url_basename* fields, but they are obtained with
        a server request.  This is useful if the file name is set in the
        Content-Disposition header.


    *Examples*

    Consider a file "avatars.csv" that contains::

        who,ext,link
        neurodebian,png,https://avatars3.githubusercontent.com/u/260793
        datalad,png,https://avatars1.githubusercontent.com/u/8927200

    To download each link into a file name composed of the 'who' and 'ext'
    fields, we could run::

      $ datalad addurls -d avatar_ds --fast avatars.csv '{link}' '{who}.{ext}'

    The `-d avatar_ds` is used to create a new dataset in "$PWD/avatar_ds".

    If we were already in a dataset and wanted to create a new subdataset in an
    "avatars" subdirectory, we could use "//" in the `FILENAME-FORMAT`
    argument::

      $ datalad addurls --fast avatars.csv '{link}' 'avatars//{who}.{ext}'

    If the information is represented as JSON lines instead of comma separated
    values or a JSON array, you can use a utility like jq to transform the JSON
    lines into an array that addurls accepts::

      $ ... | jq --slurp . | datalad addurls - '{link}' '{who}.{ext}'

    .. note::

       For users familiar with 'git annex addurl': A large part of this
       plugin's functionality can be viewed as transforming data from
       `URL-FILE` into a "url filename" format that fed to 'git annex addurl
       --batch --with-files'.
    """

    from datalad.distribution.dataset import datasetmethod
    from datalad.interface.utils import eval_results
    from datalad.distribution.dataset import EnsureDataset
    from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr
    from datalad.support.param import Parameter

    _params_ = dict(
        dataset=Parameter(
            args=("-d", "--dataset"),
            doc="""Add the URLs to this dataset (or possibly subdatasets of
            this dataset).  An empty or non-existent directory is passed to
            create a new dataset.  New subdatasets can be specified with
            `FILENAME-FORMAT`.""",
            constraints=EnsureDataset() | EnsureNone()),
        urlfile=Parameter(
            args=("urlfile",),
            metavar="URL-FILE",
            doc="""A file that contains URLs or information that can be used to
            construct URLs.  Depending on the value of --input-type, this
            should be a comma- or tab-separated file (with a header as the
            first row) or a JSON file (structured as a list of objects with
            string values). If '-', read from standard input, taking the
            content as JSON when --input-type is at its default value of
            'ext'."""),
        urlformat=Parameter(
            args=("urlformat",),
            metavar="URL-FORMAT",
            doc="""A format string that specifies the URL for each entry.  See
            the 'Format Specification' section above."""),
        filenameformat=Parameter(
            args=("filenameformat",),
            metavar="FILENAME-FORMAT",
            doc="""Like `URL-FORMAT`, but this format string specifies the file
            to which the URL's content will be downloaded. The name should be a
            relative path and will be taken as relative to the top-level
            dataset, regardless of whether it is specified via [PY: `dataset`
            PY][CMD: --dataset CMD] or inferred. The file name may contain
            directories. The separator "//" can be used to indicate that the
            left-side directory should be created as a new subdataset. See the
            'Format Specification' section above."""),
        input_type=Parameter(
            args=("-t", "--input-type"),
            metavar="TYPE",
            doc="""Whether `URL-FILE` should be considered a CSV file, TSV
            file, or JSON file. The default value, "ext", means to consider
            `URL-FILE` as a JSON file if it ends with ".json" or a TSV file if
            it ends with ".tsv". Otherwise, treat it as a CSV file.""",
            constraints=EnsureChoice(*INPUT_TYPES)),
        exclude_autometa=Parameter(
            args=("-x", "--exclude_autometa"),
            metavar="REGEXP",
            doc="""By default, metadata field=value pairs are constructed with
            each column in `URL-FILE`, excluding any single column that is
            specified via `URL-FORMAT`.  This argument can be used to exclude
            columns that match a regular expression.  If set to '*' or an empty
            string, automatic metadata extraction is disabled completely.  This
            argument does not affect metadata set explicitly with --meta."""),
        meta=Parameter(
            args=("-m", "--meta",),
            metavar="FORMAT",
            action="append",
            doc="""A format string that specifies metadata.  It should be
            structured as "<field>=<value>".  As an example, "location={3}"
            would mean that the value for the "location" metadata field should
            be set the value of the fourth column.  This option can be given
            multiple times."""),
        key=Parameter(
            args=("--key",),
            metavar="FORMAT",
            doc="""A format string that specifies an annex key for the file
            content. In this case, the file is not downloaded; instead the key
            is used to create the file without content. The value should be
            structured as "[et:]<input backend>[-s<bytes>]--<hash>". The
            optional "et:" prefix, which requires git-annex 8.20201116 or
            later, signals to toggle extension state of the input backend
            (i.e., MD5 vs MD5E). As an example, "et:MD5-s{size}--{md5sum}"
            would use the 'md5sum' and 'size' columns to construct the key,
            migrating the key from MD5 to MD5E, with an extension based on the
            file name. Note: If the *input* backend itself is an annex
            extension backend (i.e., a backend with a trailing "E"), the key's
            extension will not be updated to match the extension of the
            corresponding file name. Thus, unless the input keys and file names
            are generated from git-annex, it is recommended to avoid using
            extension backends as input. If an extension is desired, use the
            plain variant as input and prepend "et:" so that git-annex will
            migrate from the plain backend to the extension variant."""),
        message=Parameter(
            args=("--message",),
            metavar="MESSAGE",
            doc="""Use this message when committing the URL additions.""",
            constraints=EnsureNone() | EnsureStr()),
        dry_run=Parameter(
            args=("-n", "--dry-run"),
            action="store_true",
            doc="""Report which URLs would be downloaded to which files and
            then exit."""),
        fast=Parameter(
            args=("--fast",),
            action="store_true",
            doc="""If True, add the URLs, but don't download their content.
            Underneath, this passes the --fast flag to `git annex addurl`."""),
        ifexists=Parameter(
            args=("--ifexists",),
            doc="""What to do if a constructed file name already exists.  The
            default behavior is to proceed with the `git annex addurl`, which
            will fail if the file size has changed.  If set to 'overwrite',
            remove the old file before adding the new one.  If set to 'skip',
            do not add the new file.""",
            constraints=EnsureChoice(None, "overwrite", "skip")),
        missing_value=Parameter(
            args=("--missing-value",),
            metavar="VALUE",
            doc="""When an empty string is encountered, use this value
            instead.""",
            constraints=EnsureNone() | EnsureStr()),
        save=nosave_opt,
        version_urls=Parameter(
            args=("--version-urls",),
            action="store_true",
            doc="""Try to add a version ID to the URL. This currently only has
            an effect on HTTP URLs for AWS S3 buckets. s3:// URL versioning is
            not yet supported, but any URL that already contains a "versionId="
            parameter will be used as is."""),
        cfg_proc=Parameter(
            args=("-c", "--cfg-proc"),
            metavar="PROC",
            action='append',
            doc="""Pass this [PY: cfg_proc PY][CMD: --cfg_proc CMD] value when
            calling `create` to make datasets."""),
        jobs=jobs_opt,
        drop_after=Parameter(
            args=("--drop-after",),
            action="store_true",
            doc="""drop files after adding to annex""",
        ),
    )

    @staticmethod
    @datasetmethod(name='addurls')
    @eval_results
    def __call__(dataset, urlfile, urlformat, filenameformat,
                 input_type="ext", exclude_autometa=None, meta=None, key=None,
                 message=None, dry_run=False, fast=False, ifexists=None,
                 missing_value=None, save=True, version_urls=False,
                 cfg_proc=None, jobs=None, drop_after=False):
        # Temporarily work around gh-2269.
        url_file = urlfile
        url_format, filename_format = urlformat, filenameformat

        from requests.exceptions import RequestException

        from datalad.distribution.dataset import Dataset, require_dataset
        from datalad.support.annexrepo import AnnexRepo

        lgr = logging.getLogger("datalad.plugin.addurls")

        ds = require_dataset(dataset, check_installed=False)
        repo = ds.repo
        st_dict = get_status_dict(action="addurls", ds=ds)
        if repo and not isinstance(repo, AnnexRepo):
            yield dict(st_dict, status="error", message="not an annex repo")
            return

        if key:
            old_examinekey = external_versions["cmd:annex"] < "8.20201116"
            if old_examinekey:
                old_msg = None
                if key.startswith("et:"):
                    old_msg = ("et: prefix of `key` option requires "
                               "git-annex 8.20201116 or later")
                elif repo.is_managed_branch():
                    old_msg = ("Using `key` option on adjusted branch "
                               "requires git-annex 8.20201116 or later")
                if old_msg:
                    yield dict(st_dict, status="error", message=old_msg)
                    return

        if url_file != "-":
            url_file = str(resolve_path(url_file, dataset))

        if input_type == "ext":
            if url_file == "-":
                input_type = "json"
            else:
                extension = os.path.splitext(url_file)[1]
                if extension == ".json":
                    input_type = "json"
                elif extension == ".tsv":
                    input_type = "tsv"
                else:
                    input_type = "csv"

        fd = sys.stdin if url_file == "-" else open(url_file)
        try:
            try:
                rows, subpaths = extract(fd, input_type,
                                         url_format, filename_format,
                                         exclude_autometa, meta, key,
                                         dry_run,
                                         missing_value)
            except (ValueError, RequestException) as exc:
                yield dict(st_dict, status="error", message=exc_str(exc))
                return
        finally:
            if fd is not sys.stdin:
                fd.close()

        if not rows:
            yield dict(st_dict, status="notneeded",
                       message="No rows to process")
            return

        if len(rows) != len(set(row["filename"] for row in rows)):
            yield dict(st_dict, status="error",
                       message=("There are file name collisions; "
                                "consider using {_repindex}"))
            return

        if dry_run:
            for subpath in subpaths:
                lgr.info("Would create a subdataset at %s", subpath)
            for row in rows:
                lgr.info("Would %s %s to %s",
                         "register" if row.get("key") else "download",
                         row["url"],
                         os.path.join(ds.path, row["filename"]))
                if "meta_args" in row:
                    lgr.info("Metadata: %s",
                             sorted(u"{}={}".format(k, v)
                                    for k, v in row["meta_args"].items()))
            yield dict(st_dict, status="ok", message="dry-run finished")
            return

        if not repo:
            # Populate a new dataset with the URLs.
            yield from ds.create(
                result_xfm=None,
                return_type='generator',
                cfg_proc=cfg_proc)

        annex_options = ["--fast"] if fast else []

        # to be populated by addurls_to_ds
        files_to_add = set()
        created_subds = []

        def addurls_to_ds(args):
            """The "consumer" for ProducerConsumer parallel execution"""
            subpath, rows = args

            ds_path = ds.path  # shortcut on closure from outside

            # technically speaking, subds might be the ds
            if subpath:
                subds_path = os.path.join(ds_path, subpath)
            else:
                subds_path = ds_path

            for row in rows:
                # Add additional information that we'll need for various
                # operations.
                filename_abs = op.join(ds_path, row["filename"])
                ds_filename = op.relpath(filename_abs, subds_path)
                row.update({"filename_abs": filename_abs,
                            "ds_filename": ds_filename})

            subds = Dataset(subds_path)

            if subds.is_installed():
                lgr.debug(
                    "Not creating subdataset at existing path: %s",
                    subds_path)
            else:
                yield from subds.create(result_xfm=None,
                                        cfg_proc=cfg_proc,
                                        return_type='generator')
                created_subds.append(subpath)
            repo = subds.repo  # "expensive" so we get it once

            if version_urls:
                num_urls = len(rows)
                log_progress(lgr.info, "addurls_versionurls",
                             "Versioning %d URLs", num_urls,
                             label="Versioning URLs",
                             total=num_urls, unit=" URLs")
                for row in rows:
                    url = row["url"]
                    try:
                        # TODO: make get_versioned_url more efficient while going
                        # through the same bucket(s)
                        row["url"] = get_versioned_url(url)
                    except (ValueError, NotImplementedError) as exc:
                        # We don't expect this to happen because get_versioned_url
                        # should return the original URL if it isn't an S3 bucket.
                        # It only raises exceptions if it doesn't know how to
                        # handle the scheme for what looks like an S3 bucket.
                        lgr.warning("error getting version of %s: %s",
                                    row["url"], exc_str(exc))
                    log_progress(lgr.info, "addurls_versionurls",
                                 "Versioned result for %s: %s", url, row["url"],
                                 update=1, increment=True)
                log_progress(lgr.info, "addurls_versionurls", "Finished versioning URLs")

            subds_files_to_add = set()
            for r in _add_urls(rows, subds, repo,
                               ifexists=ifexists, options=annex_options,
                               drop_after=drop_after, by_key=key):
                if r["status"] == "ok":
                    subds_files_to_add.add(r["path"])
                yield r

            files_to_add.update(subds_files_to_add)
            pass  # end of addurls_to_ds


        # We need to group by dataset since otherwise we will initiate
        # batched annex process per each subdataset, which might be infeasible
        # in any use-case with a considerable number of subdatasets.
        # Also groupping allows us for parallelization across datasets, and avoids
        # proliferation of commit messages upon creation of each individual subdataset.

        def keyfn(d):
            # The top-level dataset has a subpath of None.
            return d.get("subpath") or ""

        # We need to serialize itertools.groupby .
        rows_by_ds = [(k, tuple(v)) for k, v in groupby_sorted(rows, key=keyfn)]

        # There could be "intermediate" subdatasets which have no rows but would need
        # their datasets created and saved, so let's add them
        add_subpaths = set(subpaths).difference(r[0] for r in rows_by_ds)
        nrows_by_ds_orig = len(rows_by_ds)
        if add_subpaths:
            for subpath in add_subpaths:
                rows_by_ds.append((subpath, tuple()))
            # and now resort them again since we added
            rows_by_ds = sorted(rows_by_ds, key=lambda r: r[0])

        # We want to provide progress overall files not just datasets
        # so our total will be just a len of rows
        def agg_files(*args, **kwargs):
            return len(rows)

        yield from ProducerConsumerProgressLog(
            rows_by_ds,
            addurls_to_ds,
            agg=agg_files,
            # It is ok to start with subdatasets since top dataset already exists
            safe_to_consume=partial(no_parentds_in_futures, skip=("", None, ".")),
            # our producer provides not only dataset paths and also rows, take just path
            producer_future_key=lambda row_by_ds: row_by_ds[0],
            jobs=jobs,
            # Logging options
            # we will be yielding all kinds of records, but of interest for progress
            # reporting only addurls on files
            # note: annexjson2result overrides 'action' with 'command' content from
            # annex, so we end up with 'addurl' even if we provide 'action'='addurls'.
            # TODO: see if it is all ok, since we might be now yielding both
            # addurls and addurl records.
            log_filter=_log_filter_addurls,
            unit="files",
            lgr=lgr,
        )

        if save:
            extra_msgs = []
            if created_subds:
                extra_msgs.append(f"{len(created_subds)} subdatasets were created")
            if extra_msgs:
                extra_msgs.append('')
            message_addurls = message or f"""\
[DATALAD] add {len(files_to_add)} files to {nrows_by_ds_orig} (sub)datasets from URLs

{os.linesep.join(extra_msgs)}
url_file='{urlfile}'
url_format='{url_format}'
filename_format='{filenameformat}'"""

            # save all in bulk
            files_to_add.update([r[0] for r in rows_by_ds])
            yield from ds.save(
                list(files_to_add),
                message=message_addurls,
                jobs=jobs,
                return_type='generator')


__datalad_plugin__ = Addurls

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user