{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 et:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\nfrom __future__ import annotations\n\nimport logging\n\nlgr = logging.getLogger('datalad.network')\n\nlgr.log(5, \"Importing support.network\")\nimport calendar\nimport email.utils\nimport os\nimport pickle\nimport re\nimport time\nfrom collections import OrderedDict\nfrom hashlib import md5\nfrom ntpath import splitdrive as win_splitdrive\nfrom os.path import dirname\nfrom os.path import join as opj\nfrom pathlib import PurePosixPath\nfrom urllib.error import URLError\nfrom urllib.parse import (\n    ParseResult,\n    parse_qsl,\n)\nfrom urllib.parse import quote as urlquote\nfrom urllib.parse import unquote as urlunquote\nfrom urllib.parse import (\n    quote,\n    urlencode,\n    urljoin,\n    urlparse,\n    urlsplit,\n    urlunparse,\n)\nfrom urllib.request import (\n    Request,\n    url2pathname,\n)\n\nimport iso8601\n\nfrom datalad import (\n    cfg,\n    consts,\n)\nfrom datalad.support.cache import lru_cache\nfrom datalad.support.exceptions import CapturedException\nfrom datalad.utils import (\n    Path,\n    PurePath,\n    ensure_bytes,\n    ensure_dir,\n    ensure_unicode,\n    map_items,\n    on_windows,\n)\n\n# !!! Lazily import requests where needed -- needs 30ms or so\n# import requests\n\n\ndef local_path_representation(path: str) -> str:\n    \"\"\"Return an OS-specific representation of a Posix-style path\n\n    With a posix path in the form of \"a/b\" this function will return \"a/b\" on\n    Unix-like operating systems and \"a\\\\b\" on Windows-style operating systems.\n    \"\"\"\n    return str(Path(path))\n\n\ndef local_url_path_representation(url_path: str) -> str:\n    \"\"\"Return an OS-specific representation of the path component in a file:-URL\n\n    With a path component like \"/c:/Windows\" (i.e. from a URL that reads\n    \"file:///c:/Windows\"), this function will return \"/c:/Windows\" on a\n    Unix-like operating systems and \"C:\\\\Windows\" on Windows-like operating\n    systems.\n    \"\"\"\n    return url2pathname(quote(url_path))\n\n\ndef local_path_from_url(url: str) -> str:\n    \"\"\"Parse the url and extract an OS-specific local path representation\"\"\"\n    return local_url_path_representation(urlparse(url).path)\n\n\ndef is_windows_path(path):\n    win_split = win_splitdrive(path)\n    # Note, that ntpath.splitdrive also deals with UNC paths. In that case\n    # the \"drive\" wouldn't be a windows drive letter followed by a colon\n    if win_split[0] and win_split[1] and \\\n            win_split[0].endswith(\":\") and len(win_split[0]) == 2:\n        # seems to be a windows path\n        return True\n    return False\n\n\ndef get_response_disposition_filename(s):\n    \"\"\"Given a string s as from HTTP Content-Disposition field in the response\n    return possibly present filename if any\n    \"\"\"\n    if not s:\n        return None\n    # If the response has Content-Disposition, try to get filename from it\n    cd = map(\n        lambda x: x.strip().split('=', 1) if '=' in x else [x.strip(), ''],\n        s.split(';')\n    )\n    # unify the key to be lower case and make it into a dict\n    cd = dict([[x[0].lower()] + x[1:] for x in cd])\n    if 'filename' in cd:\n        filename = cd['filename'].strip(\"\\\"'\")\n        return filename\n    return None\n\n\ndef get_url_disposition_filename(url, headers=None):\n    \"\"\"Get filename as possibly provided by the server in Content-Disposition\n    \"\"\"\n    if headers is None:\n        request = Request(url)\n        r = retry_urlopen(request)\n        # things are different in requests\n        if 'requests.' in str(r.__class__):\n            headers = r.headers\n        else:\n            headers = r.info()\n    else:\n        r = None\n    try:\n        return get_response_disposition_filename(headers.get('Content-Disposition', ''))\n    finally:\n        if r:\n            r.close()\n\n\ndef get_url_straight_filename(url, strip=None, allowdir=False):\n    \"\"\"Get file/dir name of the last path component of the URL\n\n    Parameters\n    ----------\n    strip: list, optional\n      If provided, listed names will not be considered and their\n      parent directory will be selected\n    allowdir: bool, optional\n      If url points to a \"directory\" (ends with /), empty string\n      would be returned unless allowdir is True, in which case the\n      name of the directory would be returned\n    \"\"\"\n    path = urlunquote(urlsplit(url).path)\n    path_parts = path.split('/')\n\n    if allowdir:\n        # strip empty ones\n        while len(path_parts) > 1 and not path_parts[-1]:\n            path_parts = path_parts[:-1]\n\n    if strip:\n        while path_parts and path_parts[-1] in strip:\n            path_parts = path_parts[:-1]\n\n    if path_parts:\n        return path_parts[-1]\n    else:\n        return None\n\n\ndef get_url_filename(url, headers=None, strip=None):\n    \"\"\"Get filename from the url, first consulting server about Content-Disposition\n    \"\"\"\n    filename = get_url_disposition_filename(url, headers)\n    if filename:\n        return filename\n    return get_url_straight_filename(url, strip=strip)\n\n\ndef get_url_response_stamp(url, response_info):\n    size, mtime = None, None\n    if 'Content-length' in response_info:\n        size = int(response_info['Content-length'])\n    if 'Last-modified' in response_info:\n        mtime = calendar.timegm(email.utils.parsedate(\n            response_info['Last-modified']))\n    return dict(size=size, mtime=mtime, url=url)\n\n\ndef get_tld(url):\n    \"\"\"Return top level domain from a url\n\n    Parameters\n    ----------\n    url : str\n    \"\"\"\n    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld\n    if not url.strip():\n        raise ValueError(\"Empty URL has no TLD\")\n    rec = urlsplit(url)\n    if not rec.netloc:\n        if not rec.scheme:\n            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'\n            return url.split('/', 1)[0]\n        else:\n            raise ValueError(\"It seems that only the scheme was provided without the net location/TLD\")\n    return rec.netloc\n\n\nfrom email.utils import (\n    mktime_tz,\n    parsedate_tz,\n)\n\n\ndef rfc2822_to_epoch(datestr):\n    \"\"\"Given rfc2822 date/time format, return seconds since epoch\"\"\"\n    return mktime_tz(parsedate_tz(datestr))\n\n\ndef iso8601_to_epoch(datestr):\n    \"\"\"Given ISO 8601 date/time format, return in seconds since epoch\n\n    iso8601 is used to parse properly the time zone information, which\n    can't be parsed with standard datetime strptime\n    \"\"\"\n    return calendar.timegm(iso8601.parse_date(datestr).utctimetuple())\n\n\ndef __urlopen_requests(url):\n    # XXX Workaround for now for ... broken code\n    if isinstance(url, Request):\n        url = url.get_full_url()\n    from requests import Session\n    return Session().get(url, stream=True)\n\n\ndef retry_urlopen(url, retries=3):\n    for t in range(retries):\n        try:\n            return __urlopen_requests(url)\n        except URLError as e:\n            lgr.warning(\"Received exception while reading %s: %s\" % (url, e))\n            if t == retries - 1:\n                # if we have reached allowed number of retries -- reraise\n                raise\n\n\ndef is_url_quoted(url):\n    \"\"\"Return whether URL looks being already quoted\n    \"\"\"\n    try:\n        url_ = urlunquote(url)\n        return url != url_\n    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)\n        # MIH: ValueError?\n        return False\n\n\ndef same_website(url_rec, u_rec):\n    \"\"\"Decide whether a link leads to external site\n\n    Parameters\n    ----------\n    url_rec: ParseResult\n      record for original url\n    u_rec: ParseResult\n      record for new url\n    \"\"\"\n    if isinstance(url_rec, str):\n        url_rec = urlparse(url_rec)\n    if isinstance(u_rec, str):\n        u_rec = urlparse(u_rec)\n    return (url_rec.netloc == u_rec.netloc)\n    # todo: collect more of sample cases.\n    # disabled below check while working on ratholeradio, since links\n    # could go to the parent and that is ok.  Figure out when it was\n    # desired not to go to the parent -- we might need explicit option\n    # and u_rec.path.startswith(url_rec.path)):\n\n\ndef dlurljoin(u_path, url):\n    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO\n    if url_rec.scheme:\n        # independent full url, so just return it\n        return url\n    if u_path.endswith('/'):  # should here be also a scheme use?\n        if url.startswith('/'):  # jump to the root\n            u_path_rec = urlparse(u_path)\n            return urljoin(urlunparse(\n                (u_path_rec.scheme, u_path_rec.netloc, '', '', '', '')), url)\n        else:\n            return os.path.join(u_path, url)\n    # TODO: recall where all this dirname came from and bring into the test\n    return urljoin(os.path.dirname(u_path) + '/', url)\n\n\n# TODO should it be a node maybe?\nclass SimpleURLStamper(object):\n    \"\"\"Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header\n    \"\"\"\n    def __init__(self, mode='full'):\n        self.mode = mode\n\n    def __call__(self, url):\n        # Extracted from above madness\n        # TODO: add mode alike to 'relaxed' where we would not\n        # care about content-disposition filename\n        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way\n        request = Request(url)\n\n        # No traffic compression since we do not know how to identify\n        # exactly either it has to be decompressed\n        # request.add_header('Accept-encoding', 'gzip,deflate')\n        #\n        # TODO: think about stamping etc -- we seems to be redoing\n        # what git-annex does for us already... not really\n        r = retry_urlopen(request)\n        try:\n            r_info = r.info()\n            r_stamp = get_url_response_stamp(url, r_info)\n\n            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)\n        finally:\n            r.close()\n\n\n# TODO:  make it consistent/clear at what stage % encoding/decoding happens!\n# now it is a mix!\n\n#\n# Useful functionality in requests.models\n#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance\n#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...\n#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc\n#\n\n@lru_cache(maxsize=100)\ndef _guess_ri_cls(ri):\n    \"\"\"Factory function which would determine which type of a ri a provided string is\"\"\"\n    TYPES = {\n        'url': URL,\n        'ssh':  SSHRI,\n        'file': PathRI,\n        'datalad': DataLadRI,\n        'git-transport': GitTransportRI,\n    }\n    if isinstance(ri, PurePath):\n        lgr.log(5, \"Detected file ri\")\n        return TYPES['file']\n    if is_windows_path(ri):\n        # OMG we got something from windows\n        lgr.log(5, \"Detected file ri\")\n        return TYPES['file']\n\n    # We assume that it is a URL and parse it. Depending on the result\n    # we might decide that it was something else ;)\n    fields = URL._pr_to_fields(urlparse(ri), guessing=True)\n    lgr.log(5, \"Parsed ri %s into fields %s\", ri, fields)\n    type_ = 'url'\n    # Special treatments\n    # file:///path should stay file:\n    if fields['scheme'] and fields['scheme'] not in {'file'} \\\n            and not fields['hostname']:\n        # transport::URL-or-path\n        if fields['path'].startswith(':'):  # there was ::\n            lgr.log(5, \"Assuming git transport style ri and returning\")\n            type_ = 'git-transport'\n        # dl+archive:... or just for ssh   hostname:path/p1\n        elif '+' not in fields['scheme']:\n            type_ = 'ssh'\n            lgr.log(5, \"Assuming ssh style ri, adjusted: %s\", fields)\n\n    if not fields['scheme'] and not fields['hostname']:\n        parts = _split_colon(ri)\n        # if no illegal for username@hostname characters in the first part and\n        # we either had username@hostname or multiple :-separated parts\n        if not set(parts[0]).intersection(set('/\\\\#')) and (\n                fields['path'] and\n                '@' in fields['path'] or\n                len(parts) > 1\n        ):\n            # user@host:path/sp1\n            # or host_name: (hence parts check)\n            # TODO: we need a regex to catch those really, parts check is not suff\n            type_ = 'ssh'\n        elif ri.startswith('//'):\n            # e.g. // or ///path\n            type_ = 'datalad'\n        else:\n            type_ = 'file'\n\n    if not fields['scheme'] and fields['hostname']:\n        # e.g. //a/path\n        type_ = 'datalad'\n\n    cls = TYPES[type_]\n    # just parse the ri according to regex matchint ssh \"ri\" specs\n    lgr.log(5, \"Detected %s ri\", type_)\n    return cls\n\n\nclass RI(object):\n    \"\"\"Resource Identifier - base class and a factory for URL, SSHRI, etc\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place).\n    Subclasses define specific collections of fields they care about in _FIELDS\n    class variable.\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    `RI` could be used as factory, whenever type of the resource is unknown and\n    must be guessed from the string representation.  One of the subclasses will be\n    provided as output, e.g.\n\n    >>> RI('http://example.com')\n    URL(hostname='example.com', netloc='example.com', scheme='http')\n    >>> RI('file://C:/Windows')\n    URL(hostname='c', netloc='C:', path='/Windows', scheme='file')\n    >>> RI('example.com:path')\n    SSHRI(hostname='example.com', path='path')\n    \"\"\"\n\n    # All of the subclasses will provide path\n    _FIELDS = (\n        'path',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __new__(cls, ri=None, **kwargs):\n        \"\"\"Used as a possible factory for known RI types\n\n        Returns\n        -------\n        RI\n           uninitialized RI object of appropriate class with _str\n           set to string representation if was provided\n\n        Raises\n        ------\n        ValueError\n          Whenever the RI type cannot be determined.\n        \"\"\"\n        if cls is RI and ri is not None:\n            # RI class was used as a factory\n            try:\n                cls = _guess_ri_cls(ri)\n            except Exception as e:\n                # when anything goes wrong here, ensure a homogeneous\n                # exception with a regular error\n                raise ValueError(\n                    f\"Could not determine resource identifier type for {ri!r}\"\n                ) from e\n\n        if cls is RI:\n            raise ValueError(\n                f\"Could not determine resource identifier type for {ri!r}\")\n\n        ri_obj = super(RI, cls).__new__(cls)\n        # Store internally original str\n        ri_obj._str = str(ri) if isinstance(ri, PurePath) else ri\n        return ri_obj\n\n    def __init__(self, ri=None, **fields):\n        \"\"\"\n        Parameters\n        ----------\n        ri: str, optional\n          String version of a resource specific for this class.  If you would like\n          a type of the resource be deduced, use RI(ri). Note that this value\n          will be passed to str(), so you do not have to cast it yourself.\n        **fields: dict, optional\n          The values for the fields defined in _FIELDS class variable.\n        \"\"\"\n        if ri and (bool(ri) == bool(fields)):\n            raise ValueError(\n                \"Specify either ri or breakdown from the fields, not both. \"\n                \"Got ri=%r, fields=%r\" % (ri, fields))\n\n        self._fields = self._get_blank_fields()\n        if ri is not None:\n            ri = str(ri)\n            fields = self._str_to_fields(ri)\n        self._set_from_fields(**fields)\n\n        # If was initialized from a string representation\n        if lgr.isEnabledFor(logging.DEBUG) and self._str is not None:\n            # well -- some ris might not unparse identically back\n            # strictly speaking, but let's assume they do\n            ri_ = self.as_str()\n            if ri != ri_:\n                lgr.debug(\"Parsed version of %s %r differs from original %r\",\n                          self.__class__.__name__, ri_, ri)\n\n    @classmethod\n    def _get_blank_fields(cls, **fields):\n        return OrderedDict(((f, fields.get(f, '')) for f in cls._FIELDS))\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self.as_str()\n        return self._str\n\n    @classmethod\n    def from_str(cls, ri_str):\n        obj = cls(**cls._str_to_fields(ri_str))\n        obj._str = ri_str\n        return obj\n\n    @property\n    def localpath(self):\n        # by default RIs point to remote locations\n        raise ValueError(\"%s points to remote location\" % self)\n\n    # Apparently doesn't quite play nicely with multiple inheritance for MixIn'\n    # of regexp based URLs\n    #@abstractmethod\n    #@classmethod\n    #def _str_to_fields(cls, ri_str):\n    #    raise NotImplementedError\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __bool__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields for %s are: %s\"\n                             % (unknown_fields, self.__class__, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(map_items(ensure_bytes, v))\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startswith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, RI):\n            other = RI(other)\n        return isinstance(other, self.__class__) and dict(other._fields) == dict(self._fields)\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    def __getattribute__(self, item):\n        if item.startswith('_') or item not in self._FIELDS:\n            return super(RI, self).__getattribute__(item)\n        else:\n            return self._fields[item]\n\n    def __setattr__(self, item, value):\n        if item.startswith('_') or item not in self._FIELDS:\n            super(RI, self).__setattr__(item, value)\n        else:\n            self._fields[item] = value\n            self._str = None\n\n\nclass URL(RI):\n    \"\"\"Universal resource locator\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    \"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'scheme',\n        'netloc',\n        'username',\n        'password',\n        'hostname', 'port',\n        'query',\n        'fragment',\n    )\n\n    # Only interpreted on Windows. If set to `True`, UNC-names encoded in\n    # file-URLs, e.g. \"file://server/share/path\", would be considered local\n    # and mapped onto \"\\\\server\\share\\path\". If set to `False`, UNC-names\n    # encoded in file-URLs would not be considered local and cannot be resolved\n    # to a local path.\n    support_unc = False\n\n    def as_str(self):\n        \"\"\"Render URL as a string\"\"\"\n        return urlunparse(self.to_pr())\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        fields = URL._pr_to_fields(urlparse(url_str))\n        fields['path'] = urlunquote(fields['path'])\n        return fields\n\n    def to_pr(self):\n        \"\"\"Convert URL to urlparse.ParseResults namedtuple\"\"\"\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            if fields['hostname'].count(':') >= 2:\n                # ipv6 -- need to enclose in []\n                netloc = '[%s]:%s' % (netloc, fields['port'])\n            else:\n                netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n        # We need to quote the path\n        pr_fields['path'] = urlquote(pr_fields['path'])\n        # TODO: figure out what to do with query/fragment... one step at a time\n        return ParseResult(**pr_fields)\n\n    @classmethod\n    def _pr_to_fields(cls, pr, guessing=False):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n        if pr.params:\n            (lgr.debug if guessing else lgr.warning)(\n                \"ParseResults contains params %s, which will be ignored\",\n                repr(pr.params),)\n\n        hostname_port = pr.netloc.split('@')[-1]\n        is_ipv6 = hostname_port.count(':') >= 2\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        if is_ipv6:\n            rem = re.match(r'\\[(?P<hostname>.*)\\]:(?P<port>\\d+)', hostname_port)\n            if rem:\n                hostname, port = rem.groups()\n                port = int(port)\n            else:\n                hostname, port = hostname_port, ''\n\n            def _getattr(pr, f):\n                \"\"\"Helper for custom handling in case of ipv6 addresses which blows\n                stock ParseResults logic\"\"\"\n                if f == 'port':\n                    # for now not supported at all, so\n                    return port\n                elif f == 'hostname':\n                    return hostname\n                else:\n                    return getattr(pr, f)\n        else:\n            _getattr = getattr\n\n        return {f: (_getattr(pr, f) or '') for f in cls._FIELDS}\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = map_items(ensure_unicode, OrderedDict(parse_qsl(s, 1)))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)\n\n    def _windows_local_path(self,\n                            support_unc: bool = False\n                            ) -> str:\n        \"\"\"Convert the URL to a local path on windows, supports UNC and git-annex\"\"\"\n\n        # RFC1738 and RFC3986 both forbid unescaped backslash characters in\n        # URLs, and therefore also in the path-component of file:-URLs. We\n        # assume here that any backslash present in a file-URL is a relict of a\n        # verbatim copy of a Windows-style path.\n        unified_path = self.path.replace('\\\\', '/')\n        local_path = url2pathname(unified_path)\n\n        # We support UNC notation, and the \"special\" git-annex drive encoding\n        # scheme, i.e. netloc is the drive letter plus a colon.\n        # NB, this if clause will not evaluate to True, because our caller\n        # filters out net locations with\n        if self.netloc:\n            if re.match('^[a-zA-Z]:$', self.netloc):\n                # This is the git-annex case, i.e. drive spec in netloc\n                return self.netloc + local_path\n            if support_unc:\n                return '\\\\\\\\' + self.netloc + local_path\n            raise ValueError(\"Unsupported file: URL: {self}\")\n        return local_path\n\n    @property\n    def localpath(self):\n        if self.scheme != 'file':\n            raise ValueError(\n                \"Non 'file://' URL cannot be resolved to a local path\")\n\n        # If there is a hostname, we might want to convert to UNC, unless the\n        # hostname has the form of a 'Windows drive letter' on windows\n        hostname = self.hostname\n        if not (hostname in (None, '', 'localhost', '::1')\n                or hostname.startswith('127.')\n                or re.match('^[a-zA-Z]:$', self.netloc)):\n            if on_windows:\n                return self._windows_local_path(support_unc=self.support_unc)\n            raise ValueError(\"file:// URL does not point to 'localhost'\")\n\n        if on_windows:\n            return self._windows_local_path(support_unc=True)\n\n        return url2pathname(self.path)\n\n\nclass PathRI(RI):\n    \"\"\"RI pointing to a (local) file/directory\"\"\"\n    def as_str(self):\n        return self.path\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        # str() to be compatible with pathlib objects\n        return dict(path=str(url_str))\n\n    @property\n    def localpath(self):\n        return str(Path(self.path))\n\n    @property\n    def posixpath(self):\n        if is_windows_path(self.path):\n            win_split = win_splitdrive(self.path)\n            return \"/\" + win_split[0][0] + win_split[1].replace('\\\\', '/')\n        else:\n            return self.path\n\n\nclass RegexBasedURLMixin(object):\n    \"\"\"Base class for URLs which we could simple parse using regular expressions\"\"\"\n\n    _REGEX = None\n\n    # not used ATM but possible ;)\n    # @classmethod\n    # def is_str_matches(cls, url_str):\n    #     return bool(cls._REGEX.match(url_str))\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        re_match = cls._REGEX.match(url_str)\n        if not re_match:\n            # TODO: custom error?\n            raise ValueError(\n                \"Cannot handle URL '%s': categorized as %r, but does not match syntax.%s\"\n                % (url_str,\n                   cls,\n                   \" Did you intent to use '///'?\" if url_str.startswith('//') else '')\n            )\n        fields = cls._get_blank_fields()\n        fields.update({k: v for k, v in re_match.groupdict().items() if v})\n        cls._normalize_fields(fields)\n        return fields\n\n    @classmethod\n    def _normalize_fields(self, fields):\n        \"\"\"Helper to be ran if any of the fields need to be normalized after parsing\"\"\"\n        pass\n\n\nclass SSHRI(RI, RegexBasedURLMixin):\n    \"\"\"RI pointing to a remote location reachable via SSH\"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'username',\n        'hostname',\n        'port',\n    )\n\n    _REGEX = re.compile(r'((?P<username>\\S*)@)?(?P<hostname>[^#/\\\\:]+)(\\:(?P<path>.*))?$')\n\n    @classmethod\n    def _normalize_fields(cls, fields):\n        if fields['path'] and fields['path'].startswith('//'):\n            # Let's normalize for now to avoid multiple leading slashes\n            fields['path'] = '/' + fields['path'].lstrip('/')\n        # escape path so we have direct representation of the path to work with\n        fields['path'] = unescape_ssh_path(fields['path'])\n\n    def as_str(self, escape=False):\n        fields = self.fields  # copy so we could escape symbols\n        url_fmt = '{hostname}'\n        if fields['username']:\n            url_fmt = \"{username}@\" + url_fmt\n        if fields['path']:\n            url_fmt += ':{path}'\n        if escape:\n            fields['path'] = escape_ssh_path(fields['path'])\n        return url_fmt.format(**fields)\n\n    # TODO:\n    # we can \"support\" localhost:path as localpaths\n\n\nclass DataLadRI(RI, RegexBasedURLMixin):\n    \"\"\"RI pointing to datasets within default DataLad super-dataset\"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'remote',\n    )\n\n    # For now or forever we don't deal with any fragments or other special stuff\n    _REGEX = re.compile(r'//(?P<remote>[^\\s/]*)/(?P<path>.*)$')\n\n    # do they need to be normalized??? losing track ...\n\n    def as_str(self):\n        return \"//{remote}/{path}\".format(**self._fields)\n\n    def as_git_url(self):\n        \"\"\"Dereference /// into original URLs which could be used by git for cloning\n\n        Returns\n        -------\n        str\n          URL string to reference the DataLadRI from its /// form\n        \"\"\"\n        if self.remote:\n            raise NotImplementedError(\"not supported ATM to reference additional remotes\")\n        return \"{}{}\".format(consts.DATASETS_TOPURL, urlquote(self.path))\n\n\nclass GitTransportRI(RI, RegexBasedURLMixin):\n    \"\"\"RI for some other RI with git transport prefix\"\"\"\n\n    # TODO: check how crticial to \"inherit\" RI._FIELDS asking to provide path\n    _FIELDS = RI._FIELDS + (\n        'transport',\n        'RI',\n    )\n\n    # Due to poor design, `ri` argument already present in various\n    # places intermixed with **kwargs treatment. So we will use RI\n    # here instead of ri.\n    _REGEX = re.compile(r'(?P<transport>[A-Za-z0-9][A-Za-z0-9+.-]*)::(?P<RI>.*)$')\n\n    def as_str(self):\n        return '{self.transport}::{self.RI}'.format(self=self)\n\n\ndef _split_colon(s, maxsplit=1):\n    \"\"\"Split on unescaped colon\"\"\"\n    return re.compile(r'(?<!\\\\):').split(s, maxsplit=maxsplit)\n\n# \\ should be first to deal with\n_SSH_ESCAPED_CHARACTERS = '\\\\#&;`|*?~<>^()[]{}$\\'\" '\n\n\n# TODO: RF using re.sub\ndef escape_ssh_path(path):\n    \"\"\"Escape all special characters present in the path\"\"\"\n    for c in _SSH_ESCAPED_CHARACTERS:\n        if c in path:\n            path = path.replace(c, '\\\\' + c)\n    return path\n\n\ndef unescape_ssh_path(path):\n    \"\"\"Un-escape all special characters present in the path\"\"\"\n    for c in _SSH_ESCAPED_CHARACTERS[::-1]:\n        if c in path:\n            path = path.replace('\\\\' + c, c)\n    return path\n\n\ndef parse_url_opts(url):\n    \"\"\"Given a string with url-style query, split into content before # and options as dict\"\"\"\n    url = URL(url)\n    # we need to filter out query and fragment to get the base url\n    fields = url.fields\n    fields.pop('query')\n    fields.pop('fragment')\n    opts = url.query_dict\n    return str(URL(**fields)), opts\n\n\n# TODO: should we just define URL.good_for_git or smth like that? ;)\n# although git also understands regular paths\ndef is_url(ri):\n    \"\"\"Returns whether argument is a resource identifier what datalad should treat as a URL\n\n    This includes ssh \"urls\" which git understands.\n\n    Parameters\n    ----------\n    ri : str or RI\n      The resource identifier (as a string or RI) to \"analyze\"\n    \"\"\"\n    if not isinstance(ri, RI):\n        try:\n            ri = RI(ri)\n        except:  # MIH: MemoryError?\n            return False\n    return isinstance(ri, (URL, SSHRI))\n\n\n# TODO: RF to remove duplication\ndef is_datalad_compat_ri(ri):\n    \"\"\"Returns whether argument is a resource identifier what datalad should treat as a URL\n\n    including its own DataLadRI\n    \"\"\"\n    if not isinstance(ri, RI):\n        try:\n            ri = RI(ri)\n        except:  # MIH: MemoryError?\n            return False\n    return isinstance(ri, (URL, SSHRI, DataLadRI))\n\n\n# TODO: better name? additionally may be move to SSHRI.is_valid() or sth.\ndef is_ssh(ri):\n    \"\"\"helper to determine, whether `ri` requires an SSH connection\n\n    Parameters\n    ----------\n    ri: str or RI\n\n    Returns\n    -------\n    bool\n    \"\"\"\n\n    # not exactly fitting the doc, but we actually can deal not necessarily with\n    # string or RI only, but with everything RI itself can deal with:\n    _ri = RI(ri) if not isinstance(ri, RI) else ri\n\n    return isinstance(_ri, SSHRI) \\\n        or (isinstance(_ri, URL) and _ri.scheme == 'ssh')\n\n\ndef get_local_file_url(fname: str,\n                       compatibility: str = 'git-annex',\n                       allow_relative_path: bool = True\n                       ) -> str:\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    compatibility : str, optional\n        This parameter is only interpreted on Windows systems. If set to\n        anything else than 'git', the anchor, e.g. `C:` of `fname` will be put\n        into the `file-auth` part, i.e. network location, defined in RFC 8089.\n        This option is mainly used to support git-annex specific encoding of\n        Windows paths.\n    allow_relative_path: bool, optional\n        Allow `fname` to be a relative path. The path will be converted to an\n        absolute path, by using the current directory as path prefix.\n    \"\"\"\n    path_obj = Path(fname)\n    if not path_obj.is_absolute():\n        if allow_relative_path:\n            fname = str(Path(os.getcwd()) / path_obj)\n        else:\n            raise ValueError('cannot create file-URL with relative path')\n\n    url_path = local_path2url_path(fname, auto_resolve=False)\n    if on_windows and compatibility != \"git\":\n        # Work around the way in which git-annex interprets file URLs on\n        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`\n        # into the network location component of the resulting URL.\n        return \"file:/\" + url_path\n\n    result = \"file://\" + url_path\n    if result != Path(fname).as_uri():\n        lgr.warning(\n            f\"get_local_file_url({fname}, {compatibility}, \"\n            f\"{allow_relative_path}) returned {result} instead of \"\n            f\"{Path(fname).as_uri()}\")\n    return result\n\n\ndef get_url_cache_filename(url, name=None):\n    \"\"\"Return a filename where to cache online doc from a url\"\"\"\n    if not name:\n        name = \"misc\"\n    cache_dir = opj(cfg.obtain('datalad.locations.cache'), name)\n    doc_fname = opj(\n        cache_dir,\n        '{}-{}.p{}'.format(\n            urlsplit(url).netloc,\n            md5(url.encode('utf-8')).hexdigest(),\n            pickle.HIGHEST_PROTOCOL)\n    )\n    return doc_fname\n\n\ndef get_cached_url_content(url, name=None, fetcher=None, maxage=None):\n    \"\"\"Loader of a document from a url, which caches loaded instance on disk\n\n    Doesn't do anything smart about http headers etc which could provide\n    information for cache/proxy servers for how long to retain etc\n\n    TODO: theoretically it is not network specific at all -- and just a memoize\n    pattern, but may be some time we would make it treat headers etc correctly.\n    And ATM would support any URL we support via providers/downloaders\n\n    Parameters\n    ----------\n    fetcher: callable, optional\n       Function to call with url if needed to be refetched\n    maxage: float, optional\n       Age in days to retain valid for.  <0 - would retain forever.  If None -\n       would consult the config, 0 - would force to reload\n    \"\"\"\n    doc_fname = get_url_cache_filename(url, name)\n    if maxage is None:\n        maxage = float(cfg.get('datalad.locations.cache-maxage'))\n\n    doc = None\n    if os.path.exists(doc_fname) and maxage != 0:\n\n        fage = (time.time() - os.stat(doc_fname).st_mtime)/(24. * 3600)\n        if maxage < 0 or fage < maxage:\n            try:\n                lgr.debug(\"use cached request result to '%s' from %s\", url, doc_fname)\n                doc = pickle.load(open(doc_fname, 'rb'))\n            except Exception as e:  # it is OK to ignore any error and fall back on the true source\n                lgr.warning(\n                    \"cannot load cache from '%s', fall back to download: %s\",\n                    doc_fname, CapturedException(e))\n\n    if doc is None:\n        if fetcher is None:\n            from datalad.downloaders.providers import Providers\n            providers = Providers.from_config_files()\n            fetcher = providers.fetch\n\n        doc = fetcher(url)\n        ensure_dir(dirname(doc_fname))\n        # use pickle to store the entire request result dict\n        pickle.dump(doc, open(doc_fname, 'wb'))\n        lgr.debug(\"stored result of request to '%s' in %s\", url, doc_fname)\n    return doc\n\n\ndef download_url(url, dest=None, overwrite=False):\n    \"\"\"Download a file from a URL\n\n    Supports and honors any DataLad \"downloader/provider\" configuration.\n\n    Parameters\n    ----------\n    url: str\n      Source URL to download from.\n    dest: Path-like or None\n      Destination file name (file must not exist), or name of a target\n      directory (must exists, and filename must be derivable from `url`).\n      If None, the downloaded content will be returned as a string.\n    overwrite: bool\n      Force overwriting an existing destination file.\n\n    Returns\n    -------\n    str\n      Path of the downloaded file, or URL content if `dest` is None.\n\n    Raises\n    ------\n    DownloadError\n      If `dest` already exists and is a file, or if `dest` is a directory\n      and no filename could be determined from `url`, or if no file was\n      found at the given `url`.\n    \"\"\"\n    from datalad.downloaders.providers import Providers\n    providers = Providers.from_config_files()\n    if dest:\n        return providers.download(url, path=str(dest), overwrite=overwrite)\n    else:\n        return providers.fetch(url)\n\n\ndef local_path2url_path(local_path: str,\n                        auto_resolve: bool = False\n                        ) -> str:\n    \"\"\"Convert a local path into a URL path component\"\"\"\n    if not local_path:\n        if not auto_resolve:\n            raise ValueError(\"cannot convert empty local path to URL path\")\n        local_path = os.getcwd()\n\n    url = urlparse(Path(local_path).as_uri())\n    if url.netloc:\n        raise ValueError(\n            f\"cannot convert remote path to an URL path: {local_path}\")\n    return url.path\n\n\ndef url_path2local_path(url_path: str | PurePosixPath,\n                        make_absolute: bool = True,\n                        ) -> str | Path:\n\n    if isinstance(url_path, PurePosixPath):\n        return_path = True\n        url_path = str(url_path)\n    else:\n        return_path = False\n\n    if not url_path or not url_path.startswith(\"/\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # path must begin with a slash.\n        raise ValueError(\n            f\"url path does not start with '/': {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    if url_path.startswith(\"//\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # first segment must not be empty, i.e. the path must not start with\n        # two or more slashes.\n        raise ValueError(\n            f\"url path has empty first segment: {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    return (\n        Path(url2pathname(url_path))\n        if return_path\n        else url2pathname(url_path)\n    )\n\n\ndef quote_path(path: str, safe: str = \"/\") -> str:\n    \"\"\"quote the path component of a URL, takes OS specifics into account\"\"\"\n    if on_windows:\n        if re.match(\"^/[a-zA-Z]:/\", path):\n            return path[:3] + quote(path[3:], safe=safe)\n    return quote(path, safe=safe)\n\n\nlgr.log(5, \"Done importing support.network\")\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 et:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\nfrom __future__ import annotations\n\nimport logging\n\nlgr = logging.getLogger('datalad.network')\n\nlgr.log(5, \"Importing support.network\")\nimport calendar\nimport email.utils\nimport os\nimport pickle\nimport re\nimport time\nfrom collections import OrderedDict\nfrom hashlib import md5\nfrom ntpath import splitdrive as win_splitdrive\nfrom os.path import dirname\nfrom os.path import join as opj\nfrom pathlib import PurePosixPath\nfrom urllib.error import URLError\nfrom urllib.parse import (\n    ParseResult,\n    parse_qsl,\n)\nfrom urllib.parse import quote as urlquote\nfrom urllib.parse import unquote as urlunquote\nfrom urllib.parse import (\n    quote,\n    urlencode,\n    urljoin,\n    urlparse,\n    urlsplit,\n    urlunparse,\n)\nfrom urllib.request import (\n    Request,\n    url2pathname,\n)\n\nimport iso8601\n\nfrom datalad import (\n    cfg,\n    consts,\n)\nfrom datalad.support.cache import lru_cache\nfrom datalad.support.exceptions import CapturedException\nfrom datalad.utils import (\n    Path,\n    PurePath,\n    ensure_bytes,\n    ensure_dir,\n    ensure_unicode,\n    map_items,\n    on_windows,\n)\n\n# !!! Lazily import requests where needed -- needs 30ms or so\n# import requests\n\n\ndef local_path_representation(path: str) -> str:\n    \"\"\"Return an OS-specific representation of a Posix-style path\n\n    With a posix path in the form of \"a/b\" this function will return \"a/b\" on\n    Unix-like operating systems and \"a\\\\b\" on Windows-style operating systems.\n    \"\"\"\n    return str(Path(path))\n\n\ndef local_url_path_representation(url_path: str) -> str:\n    \"\"\"Return an OS-specific representation of the path component in a file:-URL\n\n    With a path component like \"/c:/Windows\" (i.e. from a URL that reads\n    \"file:///c:/Windows\"), this function will return \"/c:/Windows\" on a\n    Unix-like operating systems and \"C:\\\\Windows\" on Windows-like operating\n    systems.\n    \"\"\"\n    return url2pathname(quote(url_path))\n\n\ndef local_path_from_url(url: str) -> str:\n    \"\"\"Parse the url and extract an OS-specific local path representation\"\"\"\n    return local_url_path_representation(urlparse(url).path)\n\n\ndef is_windows_path(path):\n    win_split = win_splitdrive(path)\n    # Note, that ntpath.splitdrive also deals with UNC paths. In that case\n    # the \"drive\" wouldn't be a windows drive letter followed by a colon\n    if win_split[0] and win_split[1] and \\\n            win_split[0].endswith(\":\") and len(win_split[0]) == 2:\n        # seems to be a windows path\n        return True\n    return False\n\n\ndef get_response_disposition_filename(s):\n    \"\"\"Given a string s as from HTTP Content-Disposition field in the response\n    return possibly present filename if any\n    \"\"\"\n    if not s:\n        return None\n    # If the response has Content-Disposition, try to get filename from it\n    cd = map(\n        lambda x: x.strip().split('=', 1) if '=' in x else [x.strip(), ''],\n        s.split(';')\n    )\n    # unify the key to be lower case and make it into a dict\n    cd = dict([[x[0].lower()] + x[1:] for x in cd])\n    if 'filename' in cd:\n        filename = cd['filename'].strip(\"\\\"'\")\n        return filename\n    return None\n\n\ndef get_url_disposition_filename(url, headers=None):\n    \"\"\"Get filename as possibly provided by the server in Content-Disposition\n    \"\"\"\n    if headers is None:\n        request = Request(url)\n        r = retry_urlopen(request)\n        # things are different in requests\n        if 'requests.' in str(r.__class__):\n            headers = r.headers\n        else:\n            headers = r.info()\n    else:\n        r = None\n    try:\n        return get_response_disposition_filename(headers.get('Content-Disposition', ''))\n    finally:\n        if r:\n            r.close()\n\n\ndef get_url_straight_filename(url, strip=None, allowdir=False):\n    \"\"\"Get file/dir name of the last path component of the URL\n\n    Parameters\n    ----------\n    strip: list, optional\n      If provided, listed names will not be considered and their\n      parent directory will be selected\n    allowdir: bool, optional\n      If url points to a \"directory\" (ends with /), empty string\n      would be returned unless allowdir is True, in which case the\n      name of the directory would be returned\n    \"\"\"\n    path = urlunquote(urlsplit(url).path)\n    path_parts = path.split('/')\n\n    if allowdir:\n        # strip empty ones\n        while len(path_parts) > 1 and not path_parts[-1]:\n            path_parts = path_parts[:-1]\n\n    if strip:\n        while path_parts and path_parts[-1] in strip:\n            path_parts = path_parts[:-1]\n\n    if path_parts:\n        return path_parts[-1]\n    else:\n        return None\n\n\ndef get_url_filename(url, headers=None, strip=None):\n    \"\"\"Get filename from the url, first consulting server about Content-Disposition\n    \"\"\"\n    filename = get_url_disposition_filename(url, headers)\n    if filename:\n        return filename\n    return get_url_straight_filename(url, strip=strip)\n\n\ndef get_url_response_stamp(url, response_info):\n    size, mtime = None, None\n    if 'Content-length' in response_info:\n        size = int(response_info['Content-length'])\n    if 'Last-modified' in response_info:\n        mtime = calendar.timegm(email.utils.parsedate(\n            response_info['Last-modified']))\n    return dict(size=size, mtime=mtime, url=url)\n\n\ndef get_tld(url):\n    \"\"\"Return top level domain from a url\n\n    Parameters\n    ----------\n    url : str\n    \"\"\"\n    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld\n    if not url.strip():\n        raise ValueError(\"Empty URL has no TLD\")\n    rec = urlsplit(url)\n    if not rec.netloc:\n        if not rec.scheme:\n            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'\n            return url.split('/', 1)[0]\n        else:\n            raise ValueError(\"It seems that only the scheme was provided without the net location/TLD\")\n    return rec.netloc\n\n\nfrom email.utils import (\n    mktime_tz,\n    parsedate_tz,\n)\n\n\ndef rfc2822_to_epoch(datestr):\n    \"\"\"Given rfc2822 date/time format, return seconds since epoch\"\"\"\n    return mktime_tz(parsedate_tz(datestr))\n\n\ndef iso8601_to_epoch(datestr):\n    \"\"\"Given ISO 8601 date/time format, return in seconds since epoch\n\n    iso8601 is used to parse properly the time zone information, which\n    can't be parsed with standard datetime strptime\n    \"\"\"\n    return calendar.timegm(iso8601.parse_date(datestr).utctimetuple())\n\n\ndef __urlopen_requests(url):\n    # XXX Workaround for now for ... broken code\n    if isinstance(url, Request):\n        url = url.get_full_url()\n    from requests import Session\n    return Session().get(url, stream=True)\n\n\ndef retry_urlopen(url, retries=3):\n    for t in range(retries):\n        try:\n            return __urlopen_requests(url)\n        except URLError as e:\n            lgr.warning(\"Received exception while reading %s: %s\" % (url, e))\n            if t == retries - 1:\n                # if we have reached allowed number of retries -- reraise\n                raise\n\n\ndef is_url_quoted(url):\n    \"\"\"Return whether URL looks being already quoted\n    \"\"\"\n    try:\n        url_ = urlunquote(url)\n        return url != url_\n    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)\n        # MIH: ValueError?\n        return False\n\n\ndef same_website(url_rec, u_rec):\n    \"\"\"Decide whether a link leads to external site\n\n    Parameters\n    ----------\n    url_rec: ParseResult\n      record for original url\n    u_rec: ParseResult\n      record for new url\n    \"\"\"\n    if isinstance(url_rec, str):\n        url_rec = urlparse(url_rec)\n    if isinstance(u_rec, str):\n        u_rec = urlparse(u_rec)\n    return (url_rec.netloc == u_rec.netloc)\n    # todo: collect more of sample cases.\n    # disabled below check while working on ratholeradio, since links\n    # could go to the parent and that is ok.  Figure out when it was\n    # desired not to go to the parent -- we might need explicit option\n    # and u_rec.path.startswith(url_rec.path)):\n\n\ndef dlurljoin(u_path, url):\n    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO\n    if url_rec.scheme:\n        # independent full url, so just return it\n        return url\n    if u_path.endswith('/'):  # should here be also a scheme use?\n        if url.startswith('/'):  # jump to the root\n            u_path_rec = urlparse(u_path)\n            return urljoin(urlunparse(\n                (u_path_rec.scheme, u_path_rec.netloc, '', '', '', '')), url)\n        else:\n            return os.path.join(u_path, url)\n    # TODO: recall where all this dirname came from and bring into the test\n    return urljoin(os.path.dirname(u_path) + '/', url)\n\n\n# TODO should it be a node maybe?\nclass SimpleURLStamper(object):\n    \"\"\"Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header\n    \"\"\"\n    def __init__(self, mode='full'):\n        self.mode = mode\n\n    def __call__(self, url):\n        # Extracted from above madness\n        # TODO: add mode alike to 'relaxed' where we would not\n        # care about content-disposition filename\n        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way\n        request = Request(url)\n\n        # No traffic compression since we do not know how to identify\n        # exactly either it has to be decompressed\n        # request.add_header('Accept-encoding', 'gzip,deflate')\n        #\n        # TODO: think about stamping etc -- we seems to be redoing\n        # what git-annex does for us already... not really\n        r = retry_urlopen(request)\n        try:\n            r_info = r.info()\n            r_stamp = get_url_response_stamp(url, r_info)\n\n            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)\n        finally:\n            r.close()\n\n\n# TODO:  make it consistent/clear at what stage % encoding/decoding happens!\n# now it is a mix!\n\n#\n# Useful functionality in requests.models\n#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance\n#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...\n#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc\n#\n\n@lru_cache(maxsize=100)\ndef _guess_ri_cls(ri):\n    \"\"\"Factory function which would determine which type of a ri a provided string is\"\"\"\n    TYPES = {\n        'url': URL,\n        'ssh':  SSHRI,\n        'file': PathRI,\n        'datalad': DataLadRI,\n        'git-transport': GitTransportRI,\n    }\n    if isinstance(ri, PurePath):\n        lgr.log(5, \"Detected file ri\")\n        return TYPES['file']\n    if is_windows_path(ri):\n        # OMG we got something from windows\n        lgr.log(5, \"Detected file ri\")\n        return TYPES['file']\n\n    # We assume that it is a URL and parse it. Depending on the result\n    # we might decide that it was something else ;)\n    fields = URL._pr_to_fields(urlparse(ri), guessing=True)\n    lgr.log(5, \"Parsed ri %s into fields %s\", ri, fields)\n    type_ = 'url'\n    # Special treatments\n    # file:///path should stay file:\n    if fields['scheme'] and fields['scheme'] not in {'file'} \\\n            and not fields['hostname']:\n        # transport::URL-or-path\n        if fields['path'].startswith(':'):  # there was ::\n            lgr.log(5, \"Assuming git transport style ri and returning\")\n            type_ = 'git-transport'\n        # dl+archive:... or just for ssh   hostname:path/p1\n        elif '+' not in fields['scheme']:\n            type_ = 'ssh'\n            lgr.log(5, \"Assuming ssh style ri, adjusted: %s\", fields)\n\n    if not fields['scheme'] and not fields['hostname']:\n        parts = _split_colon(ri)\n        # if no illegal for username@hostname characters in the first part and\n        # we either had username@hostname or multiple :-separated parts\n        if not set(parts[0]).intersection(set('/\\\\#')) and (\n                fields['path'] and\n                '@' in fields['path'] or\n                len(parts) > 1\n        ):\n            # user@host:path/sp1\n            # or host_name: (hence parts check)\n            # TODO: we need a regex to catch those really, parts check is not suff\n            type_ = 'ssh'\n        elif ri.startswith('//'):\n            # e.g. // or ///path\n            type_ = 'datalad'\n        else:\n            type_ = 'file'\n\n    if not fields['scheme'] and fields['hostname']:\n        # e.g. //a/path\n        type_ = 'datalad'\n\n    cls = TYPES[type_]\n    # just parse the ri according to regex matchint ssh \"ri\" specs\n    lgr.log(5, \"Detected %s ri\", type_)\n    return cls\n\n\nclass RI(object):\n    \"\"\"Resource Identifier - base class and a factory for URL, SSHRI, etc\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place).\n    Subclasses define specific collections of fields they care about in _FIELDS\n    class variable.\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    `RI` could be used as factory, whenever type of the resource is unknown and\n    must be guessed from the string representation.  One of the subclasses will be\n    provided as output, e.g.\n\n    >>> RI('http://example.com')\n    URL(hostname='example.com', netloc='example.com', scheme='http')\n    >>> RI('file://C:/Windows')\n    URL(hostname='c', netloc='C:', path='/Windows', scheme='file')\n    >>> RI('example.com:path')\n    SSHRI(hostname='example.com', path='path')\n    \"\"\"\n\n    # All of the subclasses will provide path\n    _FIELDS = (\n        'path',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __new__(cls, ri=None, **kwargs):\n        \"\"\"Used as a possible factory for known RI types\n\n        Returns\n        -------\n        RI\n           uninitialized RI object of appropriate class with _str\n           set to string representation if was provided\n\n        Raises\n        ------\n        ValueError\n          Whenever the RI type cannot be determined.\n        \"\"\"\n        if cls is RI and ri is not None:\n            # RI class was used as a factory\n            try:\n                cls = _guess_ri_cls(ri)\n            except Exception as e:\n                # when anything goes wrong here, ensure a homogeneous\n                # exception with a regular error\n                raise ValueError(\n                    f\"Could not determine resource identifier type for {ri!r}\"\n                ) from e\n\n        if cls is RI:\n            raise ValueError(\n                f\"Could not determine resource identifier type for {ri!r}\")\n\n        ri_obj = super(RI, cls).__new__(cls)\n        # Store internally original str\n        ri_obj._str = str(ri) if isinstance(ri, PurePath) else ri\n        return ri_obj\n\n    def __init__(self, ri=None, **fields):\n        \"\"\"\n        Parameters\n        ----------\n        ri: str, optional\n          String version of a resource specific for this class.  If you would like\n          a type of the resource be deduced, use RI(ri). Note that this value\n          will be passed to str(), so you do not have to cast it yourself.\n        **fields: dict, optional\n          The values for the fields defined in _FIELDS class variable.\n        \"\"\"\n        if ri and (bool(ri) == bool(fields)):\n            raise ValueError(\n                \"Specify either ri or breakdown from the fields, not both. \"\n                \"Got ri=%r, fields=%r\" % (ri, fields))\n\n        self._fields = self._get_blank_fields()\n        if ri is not None:\n            ri = str(ri)\n            fields = self._str_to_fields(ri)\n        self._set_from_fields(**fields)\n\n        # If was initialized from a string representation\n        if lgr.isEnabledFor(logging.DEBUG) and self._str is not None:\n            # well -- some ris might not unparse identically back\n            # strictly speaking, but let's assume they do\n            ri_ = self.as_str()\n            if ri != ri_:\n                lgr.debug(\"Parsed version of %s %r differs from original %r\",\n                          self.__class__.__name__, ri_, ri)\n\n    @classmethod\n    def _get_blank_fields(cls, **fields):\n        return OrderedDict(((f, fields.get(f, '')) for f in cls._FIELDS))\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self.as_str()\n        return self._str\n\n    @classmethod\n    def from_str(cls, ri_str):\n        obj = cls(**cls._str_to_fields(ri_str))\n        obj._str = ri_str\n        return obj\n\n    @property\n    def localpath(self):\n        # by default RIs point to remote locations\n        raise ValueError(\"%s points to remote location\" % self)\n\n    # Apparently doesn't quite play nicely with multiple inheritance for MixIn'\n    # of regexp based URLs\n    #@abstractmethod\n    #@classmethod\n    #def _str_to_fields(cls, ri_str):\n    #    raise NotImplementedError\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __bool__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields for %s are: %s\"\n                             % (unknown_fields, self.__class__, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(map_items(ensure_bytes, v))\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startswith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, RI):\n            other = RI(other)\n        return isinstance(other, self.__class__) and dict(other._fields) == dict(self._fields)\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    def __getattribute__(self, item):\n        if item.startswith('_') or item not in self._FIELDS:\n            return super(RI, self).__getattribute__(item)\n        else:\n            return self._fields[item]\n\n    def __setattr__(self, item, value):\n        if item.startswith('_') or item not in self._FIELDS:\n            super(RI, self).__setattr__(item, value)\n        else:\n            self._fields[item] = value\n            self._str = None\n\n\nclass URL(RI):\n    \"\"\"Universal resource locator\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    \"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'scheme',\n        'netloc',\n        'username',\n        'password',\n        'hostname', 'port',\n        'query',\n        'fragment',\n    )\n\n    # Only interpreted on Windows. If set to `True`, UNC-names encoded in\n    # file-URLs, e.g. \"file://server/share/path\", would be considered local\n    # and mapped onto \"\\\\server\\share\\path\". If set to `False`, UNC-names\n    # encoded in file-URLs would not be considered local and cannot be resolved\n    # to a local path.\n    support_unc = False\n\n    def as_str(self):\n        \"\"\"Render URL as a string\"\"\"\n        return urlunparse(self.to_pr())\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        fields = URL._pr_to_fields(urlparse(url_str))\n        fields['path'] = urlunquote(fields['path'])\n        return fields\n\n    def to_pr(self):\n        \"\"\"Convert URL to urlparse.ParseResults namedtuple\"\"\"\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            if fields['hostname'].count(':') >= 2:\n                # ipv6 -- need to enclose in []\n                netloc = '[%s]:%s' % (netloc, fields['port'])\n            else:\n                netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n        # We need to quote the path\n        pr_fields['path'] = urlquote(pr_fields['path'])\n        # TODO: figure out what to do with query/fragment... one step at a time\n        return ParseResult(**pr_fields)\n\n    @classmethod\n    def _pr_to_fields(cls, pr, guessing=False):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n        if pr.params:\n            (lgr.debug if guessing else lgr.warning)(\n                \"ParseResults contains params %s, which will be ignored\",\n                repr(pr.params),)\n\n        hostname_port = pr.netloc.split('@')[-1]\n        is_ipv6 = hostname_port.count(':') >= 2\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        if is_ipv6:\n            rem = re.match(r'\\[(?P<hostname>.*)\\]:(?P<port>\\d+)', hostname_port)\n            if rem:\n                hostname, port = rem.groups()\n                port = int(port)\n            else:\n                hostname, port = hostname_port, ''\n\n            def _getattr(pr, f):\n                \"\"\"Helper for custom handling in case of ipv6 addresses which blows\n                stock ParseResults logic\"\"\"\n                if f == 'port':\n                    # for now not supported at all, so\n                    return port\n                elif f == 'hostname':\n                    return hostname\n                else:\n                    return getattr(pr, f)\n        else:\n            _getattr = getattr\n\n        return {f: (_getattr(pr, f) or '') for f in cls._FIELDS}\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = map_items(ensure_unicode, OrderedDict(parse_qsl(s, 1)))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)\n\n    def _windows_local_path(self,\n                            support_unc: bool = False\n                            ) -> str:\n        \"\"\"Convert the URL to a local path on windows, supports UNC and git-annex\"\"\"\n\n        # RFC1738 and RFC3986 both forbid unescaped backslash characters in\n        # URLs, and therefore also in the path-component of file:-URLs. We\n        # assume here that any backslash present in a file-URL is a relict of a\n        # verbatim copy of a Windows-style path.\n        unified_path = self.path.replace('\\\\', '/')\n        local_path = url2pathname(unified_path)\n\n        # We support UNC notation, and the \"special\" git-annex drive encoding\n        # scheme, i.e. netloc is the drive letter plus a colon.\n        # NB, this if clause will not evaluate to True, because our caller\n        # filters out net locations with\n        if self.netloc:\n            if re.match('^[a-zA-Z]:$', self.netloc):\n                # This is the git-annex case, i.e. drive spec in netloc\n                return self.netloc + local_path\n            if support_unc:\n                return '\\\\\\\\' + self.netloc + local_path\n            raise ValueError(\"Unsupported file: URL: {self}\")\n        return local_path\n\n    @property\n    def localpath(self):\n        if self.scheme != 'file':\n            raise ValueError(\n                \"Non 'file://' URL cannot be resolved to a local path\")\n\n        # If there is a hostname, we might want to convert to UNC, unless the\n        # hostname has the form of a 'Windows drive letter' on windows\n        hostname = self.hostname\n        if not (hostname in (None, '', 'localhost', '::1')\n                or hostname.startswith('127.')\n                or re.match('^[a-zA-Z]:$', self.netloc)):\n            if on_windows:\n                return self._windows_local_path(support_unc=self.support_unc)\n            raise ValueError(\"file:// URL does not point to 'localhost'\")\n\n        if on_windows:\n            return self._windows_local_path(support_unc=True)\n\n        return url2pathname(self.path)\n\n\nclass PathRI(RI):\n    \"\"\"RI pointing to a (local) file/directory\"\"\"\n    def as_str(self):\n        return self.path\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        # str() to be compatible with pathlib objects\n        return dict(path=str(url_str))\n\n    @property\n    def localpath(self):\n        return str(Path(self.path))\n\n    @property\n    def posixpath(self):\n        if is_windows_path(self.path):\n            win_split = win_splitdrive(self.path)\n            return \"/\" + win_split[0][0] + win_split[1].replace('\\\\', '/')\n        else:\n            return self.path\n\n\nclass RegexBasedURLMixin(object):\n    \"\"\"Base class for URLs which we could simple parse using regular expressions\"\"\"\n\n    _REGEX = None\n\n    # not used ATM but possible ;)\n    # @classmethod\n    # def is_str_matches(cls, url_str):\n    #     return bool(cls._REGEX.match(url_str))\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        re_match = cls._REGEX.match(url_str)\n        if not re_match:\n            # TODO: custom error?\n            raise ValueError(\n                \"Cannot handle URL '%s': categorized as %r, but does not match syntax.%s\"\n                % (url_str,\n                   cls,\n                   \" Did you intent to use '///'?\" if url_str.startswith('//') else '')\n            )\n        fields = cls._get_blank_fields()\n        fields.update({k: v for k, v in re_match.groupdict().items() if v})\n        cls._normalize_fields(fields)\n        return fields\n\n    @classmethod\n    def _normalize_fields(self, fields):\n        \"\"\"Helper to be ran if any of the fields need to be normalized after parsing\"\"\"\n        pass\n\n\nclass SSHRI(RI, RegexBasedURLMixin):\n    \"\"\"RI pointing to a remote location reachable via SSH\"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'username',\n        'hostname',\n        'port',\n    )\n\n    _REGEX = re.compile(r'((?P<username>\\S*)@)?(?P<hostname>[^#/\\\\:]+)(\\:(?P<path>.*))?$')\n\n    @classmethod\n    def _normalize_fields(cls, fields):\n        if fields['path'] and fields['path'].startswith('//'):\n            # Let's normalize for now to avoid multiple leading slashes\n            fields['path'] = '/' + fields['path'].lstrip('/')\n        # escape path so we have direct representation of the path to work with\n        fields['path'] = unescape_ssh_path(fields['path'])\n\n    def as_str(self, escape=False):\n        fields = self.fields  # copy so we could escape symbols\n        url_fmt = '{hostname}'\n        if fields['username']:\n            url_fmt = \"{username}@\" + url_fmt\n        if fields['path']:\n            url_fmt += ':{path}'\n        if escape:\n            fields['path'] = escape_ssh_path(fields['path'])\n        return url_fmt.format(**fields)\n\n    # TODO:\n    # we can \"support\" localhost:path as localpaths\n\n\nclass DataLadRI(RI, RegexBasedURLMixin):\n    \"\"\"RI pointing to datasets within default DataLad super-dataset\"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'remote',\n    )\n\n    # For now or forever we don't deal with any fragments or other special stuff\n    _REGEX = re.compile(r'//(?P<remote>[^\\s/]*)/(?P<path>.*)$')\n\n    # do they need to be normalized??? losing track ...\n\n    def as_str(self):\n        return \"//{remote}/{path}\".format(**self._fields)\n\n    def as_git_url(self):\n        \"\"\"Dereference /// into original URLs which could be used by git for cloning\n\n        Returns\n        -------\n        str\n          URL string to reference the DataLadRI from its /// form\n        \"\"\"\n        if self.remote:\n            raise NotImplementedError(\"not supported ATM to reference additional remotes\")\n        return \"{}{}\".format(consts.DATASETS_TOPURL, urlquote(self.path))\n\n\nclass GitTransportRI(RI, RegexBasedURLMixin):\n    \"\"\"RI for some other RI with git transport prefix\"\"\"\n\n    # TODO: check how crticial to \"inherit\" RI._FIELDS asking to provide path\n    _FIELDS = RI._FIELDS + (\n        'transport',\n        'RI',\n    )\n\n    # Due to poor design, `ri` argument already present in various\n    # places intermixed with **kwargs treatment. So we will use RI\n    # here instead of ri.\n    _REGEX = re.compile(r'(?P<transport>[A-Za-z0-9][A-Za-z0-9+.-]*)::(?P<RI>.*)$')\n\n    def as_str(self):\n        return '{self.transport}::{self.RI}'.format(self=self)\n\n\ndef _split_colon(s, maxsplit=1):\n    \"\"\"Split on unescaped colon\"\"\"\n    return re.compile(r'(?<!\\\\):').split(s, maxsplit=maxsplit)\n\n# \\ should be first to deal with\n_SSH_ESCAPED_CHARACTERS = '\\\\#&;`|*?~<>^()[]{}$\\'\" '\n\n\n# TODO: RF using re.sub\ndef escape_ssh_path(path):\n    \"\"\"Escape all special characters present in the path\"\"\"\n    for c in _SSH_ESCAPED_CHARACTERS:\n        if c in path:\n            path = path.replace(c, '\\\\' + c)\n    return path\n\n\ndef unescape_ssh_path(path):\n    \"\"\"Un-escape all special characters present in the path\"\"\"\n    for c in _SSH_ESCAPED_CHARACTERS[::-1]:\n        if c in path:\n            path = path.replace('\\\\' + c, c)\n    return path\n\n\ndef parse_url_opts(url):\n    \"\"\"Given a string with url-style query, split into content before # and options as dict\"\"\"\n    url = URL(url)\n    # we need to filter out query and fragment to get the base url\n    fields = url.fields\n    fields.pop('query')\n    fields.pop('fragment')\n    opts = url.query_dict\n    return str(URL(**fields)), opts\n\n\n# TODO: should we just define URL.good_for_git or smth like that? ;)\n# although git also understands regular paths\ndef is_url(ri):\n    \"\"\"Returns whether argument is a resource identifier what datalad should treat as a URL\n\n    This includes ssh \"urls\" which git understands.\n\n    Parameters\n    ----------\n    ri : str or RI\n      The resource identifier (as a string or RI) to \"analyze\"\n    \"\"\"\n    if not isinstance(ri, RI):\n        try:\n            ri = RI(ri)\n        except:  # MIH: MemoryError?\n            return False\n    return isinstance(ri, (URL, SSHRI))\n\n\n# TODO: RF to remove duplication\ndef is_datalad_compat_ri(ri):\n    \"\"\"Returns whether argument is a resource identifier what datalad should treat as a URL\n\n    including its own DataLadRI\n    \"\"\"\n    if not isinstance(ri, RI):\n        try:\n            ri = RI(ri)\n        except:  # MIH: MemoryError?\n            return False\n    return isinstance(ri, (URL, SSHRI, DataLadRI))\n\n\n# TODO: better name? additionally may be move to SSHRI.is_valid() or sth.\ndef is_ssh(ri):\n    \"\"\"helper to determine, whether `ri` requires an SSH connection\n\n    Parameters\n    ----------\n    ri: str or RI\n\n    Returns\n    -------\n    bool\n    \"\"\"\n\n    # not exactly fitting the doc, but we actually can deal not necessarily with\n    # string or RI only, but with everything RI itself can deal with:\n    _ri = RI(ri) if not isinstance(ri, RI) else ri\n\n    return isinstance(_ri, SSHRI) \\\n        or (isinstance(_ri, URL) and _ri.scheme == 'ssh')\n\n\ndef get_local_file_url(fname: str,\n                       compatibility: str = 'git-annex',\n                       allow_relative_path: bool = True\n                       ) -> str:\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    compatibility : str, optional\n        This parameter is only interpreted on Windows systems. If set to\n        anything else than 'git', the anchor, e.g. `C:` of `fname` will be put\n        into the `file-auth` part, i.e. network location, defined in RFC 8089.\n        This option is mainly used to support git-annex specific encoding of\n        Windows paths.\n    allow_relative_path: bool, optional\n        Allow `fname` to be a relative path. The path will be converted to an\n        absolute path, by using the current directory as path prefix.\n    \"\"\"\n    url_path = local_path2url_path(fname, allow_relative_path=allow_relative_path)\n    if on_windows and compatibility != \"git\":\n        # Work around the way in which git-annex interprets file URLs on\n        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`\n        # into the network location component of the resulting URL.\n        return \"file:/\" + url_path\n\n    result = \"file://\" + url_path\n    return result\n\n\ndef get_url_cache_filename(url, name=None):\n    \"\"\"Return a filename where to cache online doc from a url\"\"\"\n    if not name:\n        name = \"misc\"\n    cache_dir = opj(cfg.obtain('datalad.locations.cache'), name)\n    doc_fname = opj(\n        cache_dir,\n        '{}-{}.p{}'.format(\n            urlsplit(url).netloc,\n            md5(url.encode('utf-8')).hexdigest(),\n            pickle.HIGHEST_PROTOCOL)\n    )\n    return doc_fname\n\n\ndef get_cached_url_content(url, name=None, fetcher=None, maxage=None):\n    \"\"\"Loader of a document from a url, which caches loaded instance on disk\n\n    Doesn't do anything smart about http headers etc which could provide\n    information for cache/proxy servers for how long to retain etc\n\n    TODO: theoretically it is not network specific at all -- and just a memoize\n    pattern, but may be some time we would make it treat headers etc correctly.\n    And ATM would support any URL we support via providers/downloaders\n\n    Parameters\n    ----------\n    fetcher: callable, optional\n       Function to call with url if needed to be refetched\n    maxage: float, optional\n       Age in days to retain valid for.  <0 - would retain forever.  If None -\n       would consult the config, 0 - would force to reload\n    \"\"\"\n    doc_fname = get_url_cache_filename(url, name)\n    if maxage is None:\n        maxage = float(cfg.get('datalad.locations.cache-maxage'))\n\n    doc = None\n    if os.path.exists(doc_fname) and maxage != 0:\n\n        fage = (time.time() - os.stat(doc_fname).st_mtime)/(24. * 3600)\n        if maxage < 0 or fage < maxage:\n            try:\n                lgr.debug(\"use cached request result to '%s' from %s\", url, doc_fname)\n                doc = pickle.load(open(doc_fname, 'rb'))\n            except Exception as e:  # it is OK to ignore any error and fall back on the true source\n                lgr.warning(\n                    \"cannot load cache from '%s', fall back to download: %s\",\n                    doc_fname, CapturedException(e))\n\n    if doc is None:\n        if fetcher is None:\n            from datalad.downloaders.providers import Providers\n            providers = Providers.from_config_files()\n            fetcher = providers.fetch\n\n        doc = fetcher(url)\n        ensure_dir(dirname(doc_fname))\n        # use pickle to store the entire request result dict\n        pickle.dump(doc, open(doc_fname, 'wb'))\n        lgr.debug(\"stored result of request to '%s' in %s\", url, doc_fname)\n    return doc\n\n\ndef download_url(url, dest=None, overwrite=False):\n    \"\"\"Download a file from a URL\n\n    Supports and honors any DataLad \"downloader/provider\" configuration.\n\n    Parameters\n    ----------\n    url: str\n      Source URL to download from.\n    dest: Path-like or None\n      Destination file name (file must not exist), or name of a target\n      directory (must exists, and filename must be derivable from `url`).\n      If None, the downloaded content will be returned as a string.\n    overwrite: bool\n      Force overwriting an existing destination file.\n\n    Returns\n    -------\n    str\n      Path of the downloaded file, or URL content if `dest` is None.\n\n    Raises\n    ------\n    DownloadError\n      If `dest` already exists and is a file, or if `dest` is a directory\n      and no filename could be determined from `url`, or if no file was\n      found at the given `url`.\n    \"\"\"\n    from datalad.downloaders.providers import Providers\n    providers = Providers.from_config_files()\n    if dest:\n        return providers.download(url, path=str(dest), overwrite=overwrite)\n    else:\n        return providers.fetch(url)\n\n\ndef local_path2url_path(local_path: str,\n                        allow_relative_path: bool = False\n                        ) -> str:\n    \"\"\"Convert a local path into an URL path component\"\"\"\n    local_path = Path(local_path)\n    if not local_path.is_absolute() and allow_relative_path:\n        local_path = local_path.absolute()\n\n    url = urlparse(Path(local_path).as_uri())\n    if url.netloc:\n        raise ValueError(\n            f\"cannot convert remote path to an URL path: {local_path}\")\n    return url.path\n\n\ndef url_path2local_path(url_path: str | PurePosixPath) -> str | Path:\n    if isinstance(url_path, PurePosixPath):\n        return_path = True\n        url_path = str(url_path)\n    else:\n        return_path = False\n\n    if not url_path or not url_path.startswith(\"/\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # path must begin with a slash.\n        raise ValueError(\n            f\"url path does not start with '/': {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    if url_path.startswith(\"//\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # first segment must not be empty, i.e. the path must not start with\n        # two or more slashes.\n        raise ValueError(\n            f\"url path has empty first segment: {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    return (\n        Path(url2pathname(url_path))\n        if return_path\n        else url2pathname(url_path)\n    )\n\n\ndef quote_path(path: str, safe: str = \"/\") -> str:\n    \"\"\"quote the path component of a URL, takes OS specifics into account\"\"\"\n    if on_windows:\n        if re.match(\"^/[a-zA-Z]:/\", path):\n            return path[:3] + quote(path[3:], safe=safe)\n    return quote(path, safe=safe)\n\n\nlgr.log(5, \"Done importing support.network\")\n",
            "header": {
                "file": {
                    "path": "datalad/support/network.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "30270a20e",
                "new_commit_hash": "856eac792",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/network.py"
            },
            "plus_file": {
                "path": "b/datalad/support/network.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1029,
                            "length": 14
                        },
                        "new_range": {
                            "start": 1029,
                            "length": 7
                        },
                        "section": "def get_local_file_url(fname: str,"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        Allow `fname` to be a relative path. The path will be converted to an",
                                "type": "context"
                            },
                            {
                                "content": "        absolute path, by using the current directory as path prefix.",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    path_obj = Path(fname)",
                                "type": "deleted"
                            },
                            {
                                "content": "    if not path_obj.is_absolute():",
                                "type": "deleted"
                            },
                            {
                                "content": "        if allow_relative_path:",
                                "type": "deleted"
                            },
                            {
                                "content": "            fname = str(Path(os.getcwd()) / path_obj)",
                                "type": "deleted"
                            },
                            {
                                "content": "        else:",
                                "type": "deleted"
                            },
                            {
                                "content": "            raise ValueError('cannot create file-URL with relative path')",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "    url_path = local_path2url_path(fname, auto_resolve=False)",
                                "type": "deleted"
                            },
                            {
                                "content": "    url_path = local_path2url_path(fname, allow_relative_path=allow_relative_path)",
                                "type": "added"
                            },
                            {
                                "content": "    if on_windows and compatibility != \"git\":",
                                "type": "context"
                            },
                            {
                                "content": "        # Work around the way in which git-annex interprets file URLs on",
                                "type": "context"
                            },
                            {
                                "content": "        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "function",
                            "name": "get_local_file_url",
                            "content": "def get_local_file_url(fname: str,\n                       compatibility: str = 'git-annex',\n                       allow_relative_path: bool = True\n                       ) -> str:\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    compatibility : str, optional\n        This parameter is only interpreted on Windows systems. If set to\n        anything else than 'git', the anchor, e.g. `C:` of `fname` will be put\n        into the `file-auth` part, i.e. network location, defined in RFC 8089.\n        This option is mainly used to support git-annex specific encoding of\n        Windows paths.\n    allow_relative_path: bool, optional\n        Allow `fname` to be a relative path. The path will be converted to an\n        absolute path, by using the current directory as path prefix.\n    \"\"\"\n    path_obj = Path(fname)\n    if not path_obj.is_absolute():\n        if allow_relative_path:\n            fname = str(Path(os.getcwd()) / path_obj)\n        else:\n            raise ValueError('cannot create file-URL with relative path')\n\n    url_path = local_path2url_path(fname, auto_resolve=False)\n    if on_windows and compatibility != \"git\":\n        # Work around the way in which git-annex interprets file URLs on\n        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`\n        # into the network location component of the resulting URL.\n        return \"file:/\" + url_path\n\n    result = \"file://\" + url_path\n    if result != Path(fname).as_uri():\n        lgr.warning(\n            f\"get_local_file_url({fname}, {compatibility}, \"\n            f\"{allow_relative_path}) returned {result} instead of \"\n            f\"{Path(fname).as_uri()}\")\n    return result",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 1012,
                            "end_lineno": 1052,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1044,
                            "length": 11
                        },
                        "new_range": {
                            "start": 1037,
                            "length": 6
                        },
                        "section": "def get_local_file_url(fname: str,"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        return \"file:/\" + url_path",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    result = \"file://\" + url_path",
                                "type": "context"
                            },
                            {
                                "content": "    if result != Path(fname).as_uri():",
                                "type": "deleted"
                            },
                            {
                                "content": "        lgr.warning(",
                                "type": "deleted"
                            },
                            {
                                "content": "            f\"get_local_file_url({fname}, {compatibility}, \"",
                                "type": "deleted"
                            },
                            {
                                "content": "            f\"{allow_relative_path}) returned {result} instead of \"",
                                "type": "deleted"
                            },
                            {
                                "content": "            f\"{Path(fname).as_uri()}\")",
                                "type": "deleted"
                            },
                            {
                                "content": "    return result",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "function",
                            "name": "get_local_file_url",
                            "content": "def get_local_file_url(fname: str,\n                       compatibility: str = 'git-annex',\n                       allow_relative_path: bool = True\n                       ) -> str:\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    compatibility : str, optional\n        This parameter is only interpreted on Windows systems. If set to\n        anything else than 'git', the anchor, e.g. `C:` of `fname` will be put\n        into the `file-auth` part, i.e. network location, defined in RFC 8089.\n        This option is mainly used to support git-annex specific encoding of\n        Windows paths.\n    allow_relative_path: bool, optional\n        Allow `fname` to be a relative path. The path will be converted to an\n        absolute path, by using the current directory as path prefix.\n    \"\"\"\n    path_obj = Path(fname)\n    if not path_obj.is_absolute():\n        if allow_relative_path:\n            fname = str(Path(os.getcwd()) / path_obj)\n        else:\n            raise ValueError('cannot create file-URL with relative path')\n\n    url_path = local_path2url_path(fname, auto_resolve=False)\n    if on_windows and compatibility != \"git\":\n        # Work around the way in which git-annex interprets file URLs on\n        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`\n        # into the network location component of the resulting URL.\n        return \"file:/\" + url_path\n\n    result = \"file://\" + url_path\n    if result != Path(fname).as_uri():\n        lgr.warning(\n            f\"get_local_file_url({fname}, {compatibility}, \"\n            f\"{allow_relative_path}) returned {result} instead of \"\n            f\"{Path(fname).as_uri()}\")\n    return result",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 1012,
                            "end_lineno": 1052,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1153,
                            "length": 13
                        },
                        "new_range": {
                            "start": 1141,
                            "length": 12
                        },
                        "section": "def download_url(url, dest=None, overwrite=False):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def local_path2url_path(local_path: str,",
                                "type": "context"
                            },
                            {
                                "content": "                        auto_resolve: bool = False",
                                "type": "deleted"
                            },
                            {
                                "content": "                        allow_relative_path: bool = False",
                                "type": "added"
                            },
                            {
                                "content": "                        ) -> str:",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"Convert a local path into a URL path component\"\"\"",
                                "type": "deleted"
                            },
                            {
                                "content": "    if not local_path:",
                                "type": "deleted"
                            },
                            {
                                "content": "        if not auto_resolve:",
                                "type": "deleted"
                            },
                            {
                                "content": "            raise ValueError(\"cannot convert empty local path to URL path\")",
                                "type": "deleted"
                            },
                            {
                                "content": "        local_path = os.getcwd()",
                                "type": "deleted"
                            },
                            {
                                "content": "    \"\"\"Convert a local path into an URL path component\"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "    local_path = Path(local_path)",
                                "type": "added"
                            },
                            {
                                "content": "    if not local_path.is_absolute() and allow_relative_path:",
                                "type": "added"
                            },
                            {
                                "content": "        local_path = local_path.absolute()",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    url = urlparse(Path(local_path).as_uri())",
                                "type": "context"
                            },
                            {
                                "content": "    if url.netloc:",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "function",
                            "name": "local_path2url_path",
                            "content": "def local_path2url_path(local_path: str,\n                        auto_resolve: bool = False\n                        ) -> str:\n    \"\"\"Convert a local path into a URL path component\"\"\"\n    if not local_path:\n        if not auto_resolve:\n            raise ValueError(\"cannot convert empty local path to URL path\")\n        local_path = os.getcwd()\n\n    url = urlparse(Path(local_path).as_uri())\n    if url.netloc:\n        raise ValueError(\n            f\"cannot convert remote path to an URL path: {local_path}\")\n    return url.path",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 1155,
                            "end_lineno": 1168,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1168,
                            "length": 10
                        },
                        "new_range": {
                            "start": 1155,
                            "length": 7
                        },
                        "section": "def local_path2url_path(local_path: str,"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    return url.path",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def url_path2local_path(url_path: str | PurePosixPath,",
                                "type": "deleted"
                            },
                            {
                                "content": "                        make_absolute: bool = True,",
                                "type": "deleted"
                            },
                            {
                                "content": "                        ) -> str | Path:",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "def url_path2local_path(url_path: str | PurePosixPath) -> str | Path:",
                                "type": "added"
                            },
                            {
                                "content": "    if isinstance(url_path, PurePosixPath):",
                                "type": "context"
                            },
                            {
                                "content": "        return_path = True",
                                "type": "context"
                            },
                            {
                                "content": "        url_path = str(url_path)",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "function",
                            "name": "url_path2local_path",
                            "content": "def url_path2local_path(url_path: str | PurePosixPath,\n                        make_absolute: bool = True,\n                        ) -> str | Path:\n\n    if isinstance(url_path, PurePosixPath):\n        return_path = True\n        url_path = str(url_path)\n    else:\n        return_path = False\n\n    if not url_path or not url_path.startswith(\"/\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # path must begin with a slash.\n        raise ValueError(\n            f\"url path does not start with '/': {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    if url_path.startswith(\"//\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # first segment must not be empty, i.e. the path must not start with\n        # two or more slashes.\n        raise ValueError(\n            f\"url path has empty first segment: {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    return (\n        Path(url2pathname(url_path))\n        if return_path\n        else url2pathname(url_path)\n    )",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 1171,
                            "end_lineno": 1200,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 et:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport logging\nimport os\nimport tempfile\nfrom collections import OrderedDict\nfrom os.path import isabs\nfrom os.path import join as opj\n\nimport pytest\n\nimport datalad.support.network\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.support.network import (\n    RI,\n    SSHRI,\n    URL,\n    DataLadRI,\n    GitTransportRI,\n    PathRI,\n    _split_colon,\n    dlurljoin,\n    get_local_file_url,\n    get_response_disposition_filename,\n    get_tld,\n    get_url_straight_filename,\n    is_datalad_compat_ri,\n    is_ssh,\n    is_url,\n    iso8601_to_epoch,\n    local_path_representation,\n    local_path2url_path,\n    local_url_path_representation,\n    parse_url_opts,\n    quote_path,\n    same_website,\n    urlquote,\n    url_path2local_path,\n)\nfrom datalad.tests.utils_pytest import (\n    OBSCURE_FILENAME,\n    SkipTest,\n    assert_in,\n    assert_raises,\n    assert_status,\n    eq_,\n    get_most_obscure_supported_name,\n    known_failure_githubci_win,\n    neq_,\n    nok_,\n    ok_,\n    skip_if,\n    swallow_logs,\n    with_tempfile,\n)\nfrom datalad.utils import (\n    Path,\n    PurePosixPath,\n    on_windows,\n)\n\n\ndef test_same_website():\n    ok_(same_website(\"http://a.b\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"https://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"https://a.b/2014/01/xxx/\"))\n\n\ndef test_get_tld():\n    eq_(get_tld('http://example.com'), 'example.com')\n    eq_(get_tld('http://example.com/1'), 'example.com')\n    eq_(get_tld('http://example.com/1/2'), 'example.com')\n    eq_(get_tld('example.com/1/2'), 'example.com')\n    eq_(get_tld('s3://example.com/1/2'), 'example.com')\n    assert_raises(ValueError, get_tld, \"\")\n    assert_raises(ValueError, get_tld, \"s3://\")\n    assert_raises(ValueError, get_tld, \"http://\")\n\ndef test_dlurljoin():\n    eq_(dlurljoin('http://a.b/', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/page', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/dir/', 'f'), 'http://a.b/dir/f')\n    eq_(dlurljoin('http://a.b/dir/', 'http://url'), 'http://url')\n    eq_(dlurljoin('http://a.b/dir/', '/'), 'http://a.b/')\n    eq_(dlurljoin('http://a.b/dir/', '/x/y'), 'http://a.b/x/y')\n\n@pytest.mark.parametrize(\"suf\", [\n    '',\n    '#',\n    '#tag',\n    '#tag/obscure',\n    '?param=1',\n    '?param=1&another=/',\n])\ndef test_get_url_straight_filename(suf):\n    eq_(get_url_straight_filename('http://a.b/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')\n\nfrom ..network import rfc2822_to_epoch\n\n\ndef test_rfc2822_to_epoch():\n    eq_(rfc2822_to_epoch(\"Thu, 16 Oct 2014 01:16:17 EDT\"), 1413436577)\n\n\ndef test_get_response_disposition_filename():\n    eq_(get_response_disposition_filename('attachment;filename=\"Part1-Subjects1-99.tar\"'), \"Part1-Subjects1-99.tar\")\n    eq_(get_response_disposition_filename('attachment'), None)\n\n\ndef test_parse_url_opts():\n    url = 'http://map.org/api/download/?id=157'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '157'}))\n\n    url = 's3://bucket/save/?key=891'\n    output = parse_url_opts(url)\n    eq_(output, ('s3://bucket/save/', {'key': '891'}))\n\n    url = 'http://map.org/api/download/?id=98&code=13'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '98', 'code': '13'}))\n\n\ndef test_split_colon():\n    eq_(_split_colon('a:b'), ['a', 'b'])\n    eq_(_split_colon('a:b:c'), ['a', 'b:c'])\n    eq_(_split_colon('a:b:c', 2), ['a', 'b', 'c'])\n    eq_(_split_colon('ab'), ['ab'])\n    eq_(_split_colon(r'a\\:b'), [r'a\\:b'])\n\n\ndef test_url_eq():\n    assert URL() == URL()\n    # doesn't make sense to ask what kind of a url it is an empty URL\n    assert URL() != URL(hostname='x')\n    # Different types aren't equal even if have the same fields values\n    assert URL(path='x') != PathRI(path='x')\n    assert URL(hostname='x') != SSHRI(hostname='x')\n    assert str(URL(hostname='x')) != str(SSHRI(hostname='x'))\n\n\ndef _check_ri(ri, cls, exact_str=True, localpath=None, **fields):\n    \"\"\"just a helper to carry out few checks on urls\"\"\"\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ri_ = cls(**fields)\n        murl = RI(ri)\n        assert murl.__class__ == cls  # not just a subclass\n        assert murl == ri_\n        if isinstance(ri, str):\n            assert str(RI(ri)) == ri\n        assert eval(repr(ri_)) == ri  # repr leads back to identical ri_\n        assert ri == ri_  # just in case ;)  above should fail first if smth is wrong\n        if not exact_str:\n            assert_in('Parsed version of', cml.out)\n    if exact_str:\n        assert str(ri) == str(ri_)\n    else:\n        assert str(ri) != str(ri_)\n\n    # and that we have access to all those fields\n    nok_(set(fields).difference(set(cls._FIELDS)))\n    for f, v in fields.items():\n        assert getattr(ri_, f) == v\n\n    if localpath:\n        if cls == URL:\n            local_representation = local_url_path_representation(localpath)\n        else:\n            local_representation = local_path_representation(localpath)\n        assert ri_.localpath == local_representation\n        old_localpath = ri_.localpath  # for a test below\n    else:\n        # if not given -- must be a remote url, should raise exception on\n        # non-Windows systems. But not on Windows systems because we allow UNCs\n        # to be encoded in URLs\n        if not on_windows:\n            with assert_raises(ValueError):\n                ri_.localpath\n\n    # This one does not have a path. TODO: either proxy path from its .RI or adjust\n    # hierarchy of classes to make it more explicit\n    if cls == GitTransportRI:\n        return\n    # do changes in the path persist?\n    old_str = str(ri_)\n    ri_.path = newpath = opj(ri_.path, 'sub')\n    assert ri_.path == newpath\n    assert str(ri_) != old_str\n    if localpath:\n        assert ri_.localpath == local_path_representation(opj(old_localpath, 'sub'))\n\n\ndef test_url_base():\n    # Basic checks\n    assert_raises(ValueError, URL, \"http://example.com\", hostname='example.com')\n    url = URL(\"http://example.com\")\n    eq_(url.hostname, 'example.com')\n    eq_(url.scheme, 'http')\n    eq_(url.port, '')  # not specified -- empty strings\n    eq_(url.username, '')  # not specified -- empty strings\n    eq_(repr(url), \"URL(hostname='example.com', netloc='example.com', scheme='http')\")\n    eq_(url, \"http://example.com\")  # automagic coercion in __eq__\n\n    neq_(URL(), URL(hostname='x'))\n\n    smth = URL('smth')\n    eq_(smth.hostname, '')\n    ok_(bool(smth))\n    nok_(bool(URL()))\n\n    assert_raises(ValueError, url._set_from_fields, unknown='1')\n\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        # we don't \"care\" about params ATM so there is a warning if there are any\n        purl = URL(\"http://example.com/;param\")\n        eq_(str(purl), 'http://example.com/;param')  # but we do maintain original string\n        assert_in('ParseResults contains params', cml.out)\n        eq_(purl.as_str(), 'http://example.com/')\n\n\n@with_tempfile\ndef test_pathri_guessing(filename=None):\n    # Complaining about ;param only at DEBUG level\n    # see https://github.com/datalad/datalad/issues/6872\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        # we don't \"care\" about params ATM so there is a warning if there are any\n        ri = RI(f\"{filename};param\")\n        assert isinstance(ri, PathRI)\n        if not on_windows:\n            # Does not happen on Windows since paths with \\ instead of / do not\n            # look like possible URLs\n            assert_in('ParseResults contains params', cml.out)\n\n\n@skip_if(not on_windows)\ndef test_pathri_windows_anchor():\n    assert RI('file:///c:/Windows').localpath == 'C:\\\\Windows'\n\n\n@known_failure_githubci_win\ndef test_url_samples():\n    _check_ri(\"http://example.com\", URL, scheme='http', hostname=\"example.com\", netloc='example.com')\n    # \"complete\" one for classical http\n    _check_ri(\"http://user:pw@example.com:8080/p/sp?p1=v1&p2=v2#frag\", URL,\n              scheme='http', netloc='user:pw@example.com:8080',\n              hostname=\"example.com\", port=8080, username='user', password='pw',\n              path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n\n    # sample one for ssh with specifying the scheme\n    # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n    _check_ri(\"ssh://host/path/sp1\", URL, scheme='ssh', hostname='host',\n              netloc='host', path='/path/sp1')\n    _check_ri(\"user@host:path/sp1\", SSHRI,\n              hostname='host', path='path/sp1', username='user')\n    _check_ri(\"host:path/sp1\", SSHRI, hostname='host', path='path/sp1')\n    _check_ri(\"host:path\", SSHRI, hostname='host', path='path')\n    _check_ri(\"host:/path\", SSHRI, hostname='host', path='/path')\n    _check_ri(\"user@host\", SSHRI, hostname='host', username='user')\n    # TODO!!!  should this be a legit URL like this?\n    # _check_ri(\"host\", SSHRI, hostname='host'))\n    eq_(repr(RI(\"host:path\")), \"SSHRI(hostname='host', path='path')\")\n\n    # And now perspective 'datalad', implicit=True urls pointing to the canonical center location\n    _check_ri(\"///\", DataLadRI)\n    _check_ri(\"///p/s1\", DataLadRI, path='p/s1')\n    # could be considered by someone as \"URI reference\" relative to scheme\n    _check_ri(\"//a/\", DataLadRI, remote='a')\n    _check_ri(\"//a/data\", DataLadRI, path='data', remote='a')\n\n    # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n    # or not so custom as\n    _check_ri(\"hg+https://host/user/proj\", URL, scheme=\"hg+https\",\n              netloc='host', hostname='host', path='/user/proj')\n    # \"old\" style\n    _check_ri(\"dl+archive:KEY/path/sp1#size=123\", URL,\n              scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n    # \"new\" style\n    _check_ri(\"dl+archive:KEY#path=path/sp1&size=123\", URL,\n              scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n    # actually above one is probably wrong since we need to encode the path\n    _check_ri(\"dl+archive:KEY#path=path%2Fbsp1&size=123\", URL,\n              scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n\n    #https://en.wikipedia.org/wiki/File_URI_scheme\n    _check_ri(\"file://host\", URL, scheme='file', netloc='host', hostname='host')\n    _check_ri(\"file://host/path/sp1\", URL, scheme='file', netloc='host',\n              hostname='host', path='/path/sp1')\n    # stock libraries of Python aren't quite ready for ipv6\n    ipv6address = '2001:db8:85a3::8a2e:370:7334'\n    _check_ri(\"file://%s/path/sp1\" % ipv6address, URL,\n              scheme='file', netloc=ipv6address, hostname=ipv6address,\n              path='/path/sp1')\n    for lh in ('localhost', '::1', '', '127.3.4.155'):\n        if on_windows:\n            url = RI(f\"file://{lh}/path/sp1\")\n            assert url.localpath == f'\\\\\\\\{lh}\\\\path\\\\sp1' if lh else '\\\\path\\\\sp1'\n        else:\n            _check_ri(\"file://%s/path/sp1\" % lh, URL, localpath='/path/sp1',\n                      scheme='file', netloc=lh, hostname=lh, path='/path/sp1')\n\n    _check_ri('http://[1fff:0:a88:85a3::ac1f]:8001/index.html', URL,\n              scheme='http', netloc='[1fff:0:a88:85a3::ac1f]:8001',\n              hostname='1fff:0:a88:85a3::ac1f', port=8001, path='/index.html')\n    _check_ri(\"file:///path/sp1\", URL, localpath='/path/sp1', scheme='file', path='/path/sp1')\n    # we don't do any magical comprehension for home paths/drives for windows\n    # of file:// urls, thus leaving /~ and /c: for now:\n    _check_ri(\"file:///~/path/sp1\", URL, localpath='/~/path/sp1', scheme='file', path='/~/path/sp1')\n    _check_ri(\"file:///%7E/path/sp1\", URL, localpath='/~/path/sp1', scheme='file', path='/~/path/sp1', exact_str=False)\n    # not sure but let's check\n    if on_windows:\n        _check_ri(\"file:///C:/path/sp1\", URL, localpath='C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        _check_ri(\"file:/C:/path/sp1\", URL, localpath='C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        # git-annex style drive-letter encoding\n        _check_ri(\"file://C:/path/sp1\", URL, netloc=\"C:\", hostname=\"c\", localpath='C:/path/sp1', scheme='file', path='/path/sp1', exact_str=False)\n    else:\n        _check_ri(\"file:///C:/path/sp1\", URL, localpath='/C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        _check_ri(\"file:/C:/path/sp1\", URL, localpath='/C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n\n    # and now implicit paths or actually they are also \"URI references\"\n    _check_ri(\"f\", PathRI, localpath='f', path='f')\n    _check_ri(\"f/s1\", PathRI, localpath='f/s1', path='f/s1')\n    _check_ri(PurePosixPath(\"f\"), PathRI, localpath='f', path='f')\n    _check_ri(PurePosixPath(\"f/s1\"), PathRI, localpath='f/s1', path='f/s1')\n    # colons are problematic and might cause confusion into SSHRI\n    _check_ri(\"f/s:1\", PathRI, localpath='f/s:1', path='f/s:1')\n    _check_ri(\"f/s:\", PathRI, localpath='f/s:', path='f/s:')\n    _check_ri(\"/f\", PathRI, localpath='/f', path='/f')\n    _check_ri(\"/f/s1\", PathRI, localpath='/f/s1', path='/f/s1')\n\n    # some github ones, just to make sure\n    _check_ri(\"git://host/user/proj\", URL, scheme=\"git\", netloc=\"host\",\n              hostname=\"host\", path=\"/user/proj\")\n    _check_ri(\"git@host:user/proj\", SSHRI, hostname=\"host\", path=\"user/proj\", username='git')\n\n    _check_ri('weird:/', SSHRI, hostname='weird', path='/')\n    # since schema is not allowing some symbols so we need to add additional check\n    _check_ri('weird_url:/', SSHRI, hostname='weird_url', path='/')\n    _check_ri('example.com:/', SSHRI, hostname='example.com', path='/')\n    _check_ri('example.com:path/sp1', SSHRI, hostname='example.com', path='path/sp1')\n    _check_ri('example.com/path/sp1\\\\:fname', PathRI, localpath='example.com/path/sp1\\\\:fname',\n              path='example.com/path/sp1\\\\:fname')\n    # ssh is as stupid as us, so we will stay \"Consistently\" dumb\n    \"\"\"\n    $> ssh example.com/path/sp1:fname\n    ssh: Could not resolve hostname example.com/path/sp1:fname: Name or service not known\n\n    edit 20190516 yoh: but this looks like a perfectly valid path.\n    SSH knows that it is not a path but its SSHRI so it can stay dumb.\n    We are trying to be smart and choose between RIs (even when we know that\n    it is e.g. a file).\n    \"\"\"\n    _check_ri('e.com/p/sp:f', PathRI, localpath='e.com/p/sp:f', path='e.com/p/sp:f')\n    _check_ri('user@someho.st/mydir', PathRI, localpath='user@someho.st/mydir', path='user@someho.st/mydir')\n\n    # SSHRIs have .port, but it is empty\n    eq_(SSHRI(hostname='example.com').port, '')\n\n    # check that we are getting a warning logged when url can't be reconstructed\n    # precisely\n    # actually failed to come up with one -- becomes late here\n    #_check_ri(\"http://host///..//p\", scheme='http', path='/..//p')\n\n    # actually this one is good enough to trigger a warning and I still don't know\n    # what it should exactly be!?\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        weird_str = 'weird://'\n        weird_url = RI(weird_str)\n        repr(weird_url)\n        cml.assert_logged(\n            'Parsed version of SSHRI .weird:/. '\n            'differs from original .weird://.'\n        )\n        # but we store original str\n        eq_(str(weird_url), weird_str)\n        neq_(weird_url.as_str(), weird_str)\n\n    raise SkipTest(\"TODO: file://::1/some does complain about parsed version dropping ::1\")\n\n\ndef test_git_transport_ri():\n    _check_ri(\"gcrypt::http://somewhere\", GitTransportRI, RI='http://somewhere', transport='gcrypt')\n    # man git-push says\n    #  <transport>::<address>\n    #    where <address> may be a path, a server and path, or an arbitrary URL-like string...\n    # so full path to my.com/... should be ok?\n    _check_ri(\"http::/my.com/some/path\", GitTransportRI, RI='/my.com/some/path', transport='http')\n    # some ssh server.  And we allow for some additional chars in transport.\n    # Git doesn't define since it does not care! we will then be flexible too\n    _check_ri(\"trans-port::server:path\", GitTransportRI, RI='server:path', transport='trans-port')\n\n\n@pytest.mark.parametrize(\"cls,clskwargs,target_url\", [\n    (SSHRI, {}, r'example.com:/ \"' + r\"';a&b&cd `| \"),\n    (URL, {'scheme': \"http\"}, 'http://example.com/%20%22%27%3Ba%26b%26cd%20%60%7C%20'),\n    (PathRI, {}, r'/ \"' + r\"';a&b&cd `| \"),  # nothing is done to file:implicit\n])\ndef test_url_quote_path(cls, clskwargs, target_url):\n    path = '/ \"\\';a&b&cd `| '\n    if not (cls is PathRI):\n        clskwargs['hostname'] = hostname = 'example.com'\n    url = cls(path=path, **clskwargs)\n    eq_(url.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n    # all nasty symbols should be quoted\n    url_str = str(url)\n    eq_(url_str, target_url)\n    # no side-effects:\n    eq_(url.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n\n    # and figured out and unquoted\n    url_ = RI(url_str)\n    ok_(isinstance(url_, cls))\n    eq_(url_.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n\n\ndef test_url_compose_archive_one():\n    url = URL(scheme='dl+archive', path='KEY',\n              fragment=OrderedDict((('path', 'f/p/ s+'), ('size', 30))))\n    # funny - space is encoded as + but + is %2B\n    eq_(str(url), 'dl+archive:KEY#path=f/p/+s%2B&size=30')\n    eq_(url.fragment_dict, {'path': 'f/p/ s+', 'size': '30'})\n\n\ndef test_url_fragments_and_query():\n    url = URL(hostname=\"host\", query=OrderedDict((('a', 'x/b'), ('b', 'y'))))\n    eq_(str(url), '//host?a=x%2Fb&b=y')\n    eq_(url.query, 'a=x%2Fb&b=y')\n    eq_(url.query_dict, {'a': 'x/b', 'b': 'y'})\n\n    url = URL(hostname=\"host\", fragment=OrderedDict((('b', 'x/b'), ('a', 'y'))))\n    eq_(str(url), '//host#b=x/b&a=y')\n    eq_(url.fragment, 'b=x/b&a=y')\n    eq_(url.fragment_dict, {'a': 'y', 'b': 'x/b'})\n\n    fname = get_most_obscure_supported_name()\n    url = URL(hostname=\"host\", fragment={'a': fname})\n    eq_(url.fragment_dict, {'a': fname})\n\n\ndef test_url_dicts():\n    eq_(URL(\"http://host\").query_dict, {})\n\n\ndef test_get_url_path_on_fileurls():\n    assert URL('file:///a').path == '/a'\n    assert URL('file:///a/b').path == '/a/b'\n    assert URL('file:///a/b').localpath == local_path_representation('/a/b')\n    assert URL('file:///a/b#id').path == '/a/b'\n    assert URL('file:///a/b?whatever').path == '/a/b'\n\n\ndef test_is_url():\n    ok_(is_url('file://localhost/some'))\n    ok_(is_url('http://localhost'))\n    ok_(is_url('ssh://me@localhost'))\n    # in current understanding it is indeed a url but an 'ssh', implicit=True, not just\n    # a useless scheme=weird with a hope to point to a netloc\n    with swallow_logs():\n        ok_(is_url('weird://'))\n    nok_(is_url('relative'))\n    nok_(is_url('/absolute'))\n    ok_(is_url('like@sshlogin'))  # actually we do allow ssh:implicit urls ATM\n    nok_(is_url(''))\n    nok_(is_url(' '))\n    nok_(is_url(123))  # stuff of other types wouldn't be considered a URL\n\n    # we can pass RI instance directly\n    ok_(is_url(RI('file://localhost/some')))\n    nok_(is_url(RI('relative')))\n\n\n# TODO: RF with test_is_url to avoid duplication\ndef test_is_datalad_compat_ri():\n    ok_(is_datalad_compat_ri('ssh://user:passw@host/path'))\n    ok_(is_datalad_compat_ri('http://example.com'))\n    ok_(is_datalad_compat_ri('file://localhost/some'))\n    ok_(is_datalad_compat_ri('///localhost/some'))\n    nok_(is_datalad_compat_ri('relative'))\n    nok_(is_datalad_compat_ri('.///localhost/some'))\n    nok_(is_datalad_compat_ri(123))\n\n\ndef test_get_local_file_url():\n    compat_annex = 'git-annex'\n    compat_git = 'git'\n    for path, url, compatibility in (\n                # relpaths are special-cased below\n                ('test.txt', 'test.txt', compat_annex),\n                (OBSCURE_FILENAME, urlquote(OBSCURE_FILENAME), compat_annex),\n            ) + (\n                ('C:\\\\Windows\\\\notepad.exe', 'file://C:/Windows/notepad.exe', compat_annex),\n                ('C:\\\\Windows\\\\notepad.exe', 'file:///C:/Windows/notepad.exe', compat_git),\n            ) if on_windows else (\n                ('/a', 'file:///a', compat_annex),\n                ('/a/b/c', 'file:///a/b/c', compat_annex),\n                ('/a~', 'file:///a~', compat_annex),\n                # there are no files with trailing slashes in the name\n                #('/a b/', 'file:///a%20b/'),\n                ('/a b/name', 'file:///a%20b/name', compat_annex),\n            ):\n        # Yarik found no better way to trigger.  .decode() isn't enough\n        print(\"D: %s\" % path)\n        if isabs(path):\n            assert get_local_file_url(path, compatibility=compatibility) == url\n        else:\n            assert get_local_file_url(path, allow_relative_path=True, compatibility=compatibility) \\\n                   == '/'.join((get_local_file_url(os.getcwd(), compatibility=compatibility), url))\n\n@with_tempfile(mkdir=True)\ndef test_get_local_file_url_compatibility(path=None):\n    # smoke test for file:// URL compatibility with other datalad/git/annex\n    # pieces\n    path = Path(path)\n    ds1 = Dataset(path / 'ds1').create()\n    ds2 = Dataset(path / 'ds2').create()\n    testfile = path / 'testfile.txt'\n    testfile.write_text('some')\n\n    # compat with annex addurl\n    ds1.repo.add_url_to_file(\n        'test.txt',\n        get_local_file_url(str(testfile), compatibility='git-annex'))\n\n    # compat with git clone/submodule\n    assert_status(\n        'ok',\n        ds1.clone(get_local_file_url(ds2.path, compatibility='git'),\n                  result_xfm=None, return_type='generator'))\n\n\ndef test_is_ssh():\n\n    ssh_locators = [\"ssh://host\",\n                    \"ssh://host/some/where\",\n                    \"user@host:path/sp1\",\n                    \"user@host:/absolute/path/sp1\",\n                    \"host:path/sp1\",\n                    \"host:/absolute/path/sp1\",\n                    \"user@host\"]\n    for ri in ssh_locators:\n        ok_(is_ssh(ri), \"not considered ssh (string): %s\" % ri)\n        ok_(is_ssh(RI(ri)), \"not considered ssh (RI): %s\" % ri)\n\n    non_ssh_locators = [\"file://path/to\",\n                        \"/abs/path\",\n                        \"../rel/path\",\n                        \"http://example.com\",\n                        \"git://host/user/proj\",\n                        \"s3://bucket/save/?key=891\"]\n    for ri in non_ssh_locators:\n        ok_(not is_ssh(ri), \"considered ssh (string): %s\" % ri)\n        ok_(not is_ssh(RI(ri)), \"considered ssh (RI): %s\" % ri)\n\n\ndef test_iso8601_to_epoch():\n    epoch = 1467901515\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15+00:00'), epoch)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15+11:00'),\n        epoch - 11 * 60 * 60)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15Z'), epoch)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15'), epoch)\n\n    eq_(iso8601_to_epoch('2016-07-07T14:25:14'), epoch-1)\n\n\ndef test_mapping_identity():\n    from datalad.tests.utils_pytest import OBSCURE_FILENAME\n\n    absolute_obscure_path = str(Path('/').absolute() / OBSCURE_FILENAME)\n    temp_dir = tempfile.gettempdir()\n    print(f\"temp_dir: {temp_dir}\")\n    for name in (temp_dir, opj(temp_dir, \"x.txt\"), absolute_obscure_path):\n        # On some platforms, e.g. MacOS, `temp_dir` might contain trailing\n        # slashes. Since the conversion and its inverse normalize paths, we\n        # compare the result to the normalized path\n        normalized_name = str(Path(name))\n        assert url_path2local_path(local_path2url_path(name)) == normalized_name\n\n    prefix = \"/C:\" if on_windows else \"\"\n    for name in map(quote_path, (prefix + \"/window\", prefix + \"/d\", prefix + \"/\" + OBSCURE_FILENAME)):\n        assert local_path2url_path(url_path2local_path(name)) == name\n\n\ndef test_auto_resolve_path():\n    relative_path = str(Path(\"a/b\"))\n    with pytest.raises(ValueError):\n        local_path2url_path(relative_path)\n    local_path2url_path(\"\", auto_resolve=True)\n\n\n@skip_if(not on_windows)\ndef test_hostname_detection():\n    with pytest.raises(ValueError):\n        local_path2url_path(\"\\\\\\\\server\\\\share\\\\path\")\n\n\ndef test_url_path2local_path_excceptions():\n    with pytest.raises(ValueError):\n        url_path2local_path('')\n    with pytest.raises(ValueError):\n        url_path2local_path(None)\n    with pytest.raises(ValueError):\n        url_path2local_path('a/b')\n    with pytest.raises(ValueError):\n        url_path2local_path(PurePosixPath('a/b'))\n    with pytest.raises(ValueError):\n        url_path2local_path(PurePosixPath('//a/b'))\n\n\ndef test_quote_path(monkeypatch):\n    with monkeypatch.context() as ctx:\n        ctx.setattr(datalad.support.network, 'on_windows', True)\n        assert quote_path(\"/c:/win:xxx\") == \"/c:/win%3Axxx\"\n        assert quote_path(\"/C:/win:xxx\") == \"/C:/win%3Axxx\"\n\n        ctx.setattr(datalad.support.network, 'on_windows', False)\n        assert quote_path(\"/c:/win:xxx\") == \"/c%3A/win%3Axxx\"\n        assert quote_path(\"/C:/win:xxx\") == \"/C%3A/win%3Axxx\"\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 et:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport logging\nimport os\nimport tempfile\nfrom collections import OrderedDict\nfrom os.path import isabs\nfrom os.path import join as opj\n\nimport pytest\n\nimport datalad.support.network\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.support.network import (\n    RI,\n    SSHRI,\n    URL,\n    DataLadRI,\n    GitTransportRI,\n    PathRI,\n    _split_colon,\n    dlurljoin,\n    get_local_file_url,\n    get_response_disposition_filename,\n    get_tld,\n    get_url_straight_filename,\n    is_datalad_compat_ri,\n    is_ssh,\n    is_url,\n    iso8601_to_epoch,\n    local_path_representation,\n    local_path2url_path,\n    local_url_path_representation,\n    parse_url_opts,\n    quote_path,\n    same_website,\n    urlquote,\n    url_path2local_path,\n)\nfrom datalad.tests.utils_pytest import (\n    OBSCURE_FILENAME,\n    SkipTest,\n    assert_in,\n    assert_raises,\n    assert_status,\n    eq_,\n    get_most_obscure_supported_name,\n    known_failure_githubci_win,\n    neq_,\n    nok_,\n    ok_,\n    skip_if,\n    swallow_logs,\n    with_tempfile,\n)\nfrom datalad.utils import (\n    Path,\n    PurePosixPath,\n    on_windows,\n)\n\n\ndef test_same_website():\n    ok_(same_website(\"http://a.b\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"https://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"https://a.b/2014/01/xxx/\"))\n\n\ndef test_get_tld():\n    eq_(get_tld('http://example.com'), 'example.com')\n    eq_(get_tld('http://example.com/1'), 'example.com')\n    eq_(get_tld('http://example.com/1/2'), 'example.com')\n    eq_(get_tld('example.com/1/2'), 'example.com')\n    eq_(get_tld('s3://example.com/1/2'), 'example.com')\n    assert_raises(ValueError, get_tld, \"\")\n    assert_raises(ValueError, get_tld, \"s3://\")\n    assert_raises(ValueError, get_tld, \"http://\")\n\ndef test_dlurljoin():\n    eq_(dlurljoin('http://a.b/', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/page', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/dir/', 'f'), 'http://a.b/dir/f')\n    eq_(dlurljoin('http://a.b/dir/', 'http://url'), 'http://url')\n    eq_(dlurljoin('http://a.b/dir/', '/'), 'http://a.b/')\n    eq_(dlurljoin('http://a.b/dir/', '/x/y'), 'http://a.b/x/y')\n\n@pytest.mark.parametrize(\"suf\", [\n    '',\n    '#',\n    '#tag',\n    '#tag/obscure',\n    '?param=1',\n    '?param=1&another=/',\n])\ndef test_get_url_straight_filename(suf):\n    eq_(get_url_straight_filename('http://a.b/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')\n\nfrom ..network import rfc2822_to_epoch\n\n\ndef test_rfc2822_to_epoch():\n    eq_(rfc2822_to_epoch(\"Thu, 16 Oct 2014 01:16:17 EDT\"), 1413436577)\n\n\ndef test_get_response_disposition_filename():\n    eq_(get_response_disposition_filename('attachment;filename=\"Part1-Subjects1-99.tar\"'), \"Part1-Subjects1-99.tar\")\n    eq_(get_response_disposition_filename('attachment'), None)\n\n\ndef test_parse_url_opts():\n    url = 'http://map.org/api/download/?id=157'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '157'}))\n\n    url = 's3://bucket/save/?key=891'\n    output = parse_url_opts(url)\n    eq_(output, ('s3://bucket/save/', {'key': '891'}))\n\n    url = 'http://map.org/api/download/?id=98&code=13'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '98', 'code': '13'}))\n\n\ndef test_split_colon():\n    eq_(_split_colon('a:b'), ['a', 'b'])\n    eq_(_split_colon('a:b:c'), ['a', 'b:c'])\n    eq_(_split_colon('a:b:c', 2), ['a', 'b', 'c'])\n    eq_(_split_colon('ab'), ['ab'])\n    eq_(_split_colon(r'a\\:b'), [r'a\\:b'])\n\n\ndef test_url_eq():\n    assert URL() == URL()\n    # doesn't make sense to ask what kind of a url it is an empty URL\n    assert URL() != URL(hostname='x')\n    # Different types aren't equal even if have the same fields values\n    assert URL(path='x') != PathRI(path='x')\n    assert URL(hostname='x') != SSHRI(hostname='x')\n    assert str(URL(hostname='x')) != str(SSHRI(hostname='x'))\n\n\ndef _check_ri(ri, cls, exact_str=True, localpath=None, **fields):\n    \"\"\"just a helper to carry out few checks on urls\"\"\"\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ri_ = cls(**fields)\n        murl = RI(ri)\n        assert murl.__class__ == cls  # not just a subclass\n        assert murl == ri_\n        if isinstance(ri, str):\n            assert str(RI(ri)) == ri\n        assert eval(repr(ri_)) == ri  # repr leads back to identical ri_\n        assert ri == ri_  # just in case ;)  above should fail first if smth is wrong\n        if not exact_str:\n            assert_in('Parsed version of', cml.out)\n    if exact_str:\n        assert str(ri) == str(ri_)\n    else:\n        assert str(ri) != str(ri_)\n\n    # and that we have access to all those fields\n    nok_(set(fields).difference(set(cls._FIELDS)))\n    for f, v in fields.items():\n        assert getattr(ri_, f) == v\n\n    if localpath:\n        if cls == URL:\n            local_representation = local_url_path_representation(localpath)\n        else:\n            local_representation = local_path_representation(localpath)\n        assert ri_.localpath == local_representation\n        old_localpath = ri_.localpath  # for a test below\n    else:\n        # if not given -- must be a remote url, should raise exception on\n        # non-Windows systems. But not on Windows systems because we allow UNCs\n        # to be encoded in URLs\n        if not on_windows:\n            with assert_raises(ValueError):\n                ri_.localpath\n\n    # This one does not have a path. TODO: either proxy path from its .RI or adjust\n    # hierarchy of classes to make it more explicit\n    if cls == GitTransportRI:\n        return\n    # do changes in the path persist?\n    old_str = str(ri_)\n    ri_.path = newpath = opj(ri_.path, 'sub')\n    assert ri_.path == newpath\n    assert str(ri_) != old_str\n    if localpath:\n        assert ri_.localpath == local_path_representation(opj(old_localpath, 'sub'))\n\n\ndef test_url_base():\n    # Basic checks\n    assert_raises(ValueError, URL, \"http://example.com\", hostname='example.com')\n    url = URL(\"http://example.com\")\n    eq_(url.hostname, 'example.com')\n    eq_(url.scheme, 'http')\n    eq_(url.port, '')  # not specified -- empty strings\n    eq_(url.username, '')  # not specified -- empty strings\n    eq_(repr(url), \"URL(hostname='example.com', netloc='example.com', scheme='http')\")\n    eq_(url, \"http://example.com\")  # automagic coercion in __eq__\n\n    neq_(URL(), URL(hostname='x'))\n\n    smth = URL('smth')\n    eq_(smth.hostname, '')\n    ok_(bool(smth))\n    nok_(bool(URL()))\n\n    assert_raises(ValueError, url._set_from_fields, unknown='1')\n\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        # we don't \"care\" about params ATM so there is a warning if there are any\n        purl = URL(\"http://example.com/;param\")\n        eq_(str(purl), 'http://example.com/;param')  # but we do maintain original string\n        assert_in('ParseResults contains params', cml.out)\n        eq_(purl.as_str(), 'http://example.com/')\n\n\n@with_tempfile\ndef test_pathri_guessing(filename=None):\n    # Complaining about ;param only at DEBUG level\n    # see https://github.com/datalad/datalad/issues/6872\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        # we don't \"care\" about params ATM so there is a warning if there are any\n        ri = RI(f\"{filename};param\")\n        assert isinstance(ri, PathRI)\n        if not on_windows:\n            # Does not happen on Windows since paths with \\ instead of / do not\n            # look like possible URLs\n            assert_in('ParseResults contains params', cml.out)\n\n\n@skip_if(not on_windows)\ndef test_pathri_windows_anchor():\n    assert RI('file:///c:/Windows').localpath == 'C:\\\\Windows'\n\n\n@known_failure_githubci_win\ndef test_url_samples():\n    _check_ri(\"http://example.com\", URL, scheme='http', hostname=\"example.com\", netloc='example.com')\n    # \"complete\" one for classical http\n    _check_ri(\"http://user:pw@example.com:8080/p/sp?p1=v1&p2=v2#frag\", URL,\n              scheme='http', netloc='user:pw@example.com:8080',\n              hostname=\"example.com\", port=8080, username='user', password='pw',\n              path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n\n    # sample one for ssh with specifying the scheme\n    # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n    _check_ri(\"ssh://host/path/sp1\", URL, scheme='ssh', hostname='host',\n              netloc='host', path='/path/sp1')\n    _check_ri(\"user@host:path/sp1\", SSHRI,\n              hostname='host', path='path/sp1', username='user')\n    _check_ri(\"host:path/sp1\", SSHRI, hostname='host', path='path/sp1')\n    _check_ri(\"host:path\", SSHRI, hostname='host', path='path')\n    _check_ri(\"host:/path\", SSHRI, hostname='host', path='/path')\n    _check_ri(\"user@host\", SSHRI, hostname='host', username='user')\n    # TODO!!!  should this be a legit URL like this?\n    # _check_ri(\"host\", SSHRI, hostname='host'))\n    eq_(repr(RI(\"host:path\")), \"SSHRI(hostname='host', path='path')\")\n\n    # And now perspective 'datalad', implicit=True urls pointing to the canonical center location\n    _check_ri(\"///\", DataLadRI)\n    _check_ri(\"///p/s1\", DataLadRI, path='p/s1')\n    # could be considered by someone as \"URI reference\" relative to scheme\n    _check_ri(\"//a/\", DataLadRI, remote='a')\n    _check_ri(\"//a/data\", DataLadRI, path='data', remote='a')\n\n    # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n    # or not so custom as\n    _check_ri(\"hg+https://host/user/proj\", URL, scheme=\"hg+https\",\n              netloc='host', hostname='host', path='/user/proj')\n    # \"old\" style\n    _check_ri(\"dl+archive:KEY/path/sp1#size=123\", URL,\n              scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n    # \"new\" style\n    _check_ri(\"dl+archive:KEY#path=path/sp1&size=123\", URL,\n              scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n    # actually above one is probably wrong since we need to encode the path\n    _check_ri(\"dl+archive:KEY#path=path%2Fbsp1&size=123\", URL,\n              scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n\n    #https://en.wikipedia.org/wiki/File_URI_scheme\n    _check_ri(\"file://host\", URL, scheme='file', netloc='host', hostname='host')\n    _check_ri(\"file://host/path/sp1\", URL, scheme='file', netloc='host',\n              hostname='host', path='/path/sp1')\n    # stock libraries of Python aren't quite ready for ipv6\n    ipv6address = '2001:db8:85a3::8a2e:370:7334'\n    _check_ri(\"file://%s/path/sp1\" % ipv6address, URL,\n              scheme='file', netloc=ipv6address, hostname=ipv6address,\n              path='/path/sp1')\n    for lh in ('localhost', '::1', '', '127.3.4.155'):\n        if on_windows:\n            url = RI(f\"file://{lh}/path/sp1\")\n            assert url.localpath == f'\\\\\\\\{lh}\\\\path\\\\sp1' if lh else '\\\\path\\\\sp1'\n        else:\n            _check_ri(\"file://%s/path/sp1\" % lh, URL, localpath='/path/sp1',\n                      scheme='file', netloc=lh, hostname=lh, path='/path/sp1')\n\n    _check_ri('http://[1fff:0:a88:85a3::ac1f]:8001/index.html', URL,\n              scheme='http', netloc='[1fff:0:a88:85a3::ac1f]:8001',\n              hostname='1fff:0:a88:85a3::ac1f', port=8001, path='/index.html')\n    _check_ri(\"file:///path/sp1\", URL, localpath='/path/sp1', scheme='file', path='/path/sp1')\n    # we don't do any magical comprehension for home paths/drives for windows\n    # of file:// urls, thus leaving /~ and /c: for now:\n    _check_ri(\"file:///~/path/sp1\", URL, localpath='/~/path/sp1', scheme='file', path='/~/path/sp1')\n    _check_ri(\"file:///%7E/path/sp1\", URL, localpath='/~/path/sp1', scheme='file', path='/~/path/sp1', exact_str=False)\n    # not sure but let's check\n    if on_windows:\n        _check_ri(\"file:///C:/path/sp1\", URL, localpath='C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        _check_ri(\"file:/C:/path/sp1\", URL, localpath='C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        # git-annex style drive-letter encoding\n        _check_ri(\"file://C:/path/sp1\", URL, netloc=\"C:\", hostname=\"c\", localpath='C:/path/sp1', scheme='file', path='/path/sp1', exact_str=False)\n    else:\n        _check_ri(\"file:///C:/path/sp1\", URL, localpath='/C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        _check_ri(\"file:/C:/path/sp1\", URL, localpath='/C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n\n    # and now implicit paths or actually they are also \"URI references\"\n    _check_ri(\"f\", PathRI, localpath='f', path='f')\n    _check_ri(\"f/s1\", PathRI, localpath='f/s1', path='f/s1')\n    _check_ri(PurePosixPath(\"f\"), PathRI, localpath='f', path='f')\n    _check_ri(PurePosixPath(\"f/s1\"), PathRI, localpath='f/s1', path='f/s1')\n    # colons are problematic and might cause confusion into SSHRI\n    _check_ri(\"f/s:1\", PathRI, localpath='f/s:1', path='f/s:1')\n    _check_ri(\"f/s:\", PathRI, localpath='f/s:', path='f/s:')\n    _check_ri(\"/f\", PathRI, localpath='/f', path='/f')\n    _check_ri(\"/f/s1\", PathRI, localpath='/f/s1', path='/f/s1')\n\n    # some github ones, just to make sure\n    _check_ri(\"git://host/user/proj\", URL, scheme=\"git\", netloc=\"host\",\n              hostname=\"host\", path=\"/user/proj\")\n    _check_ri(\"git@host:user/proj\", SSHRI, hostname=\"host\", path=\"user/proj\", username='git')\n\n    _check_ri('weird:/', SSHRI, hostname='weird', path='/')\n    # since schema is not allowing some symbols so we need to add additional check\n    _check_ri('weird_url:/', SSHRI, hostname='weird_url', path='/')\n    _check_ri('example.com:/', SSHRI, hostname='example.com', path='/')\n    _check_ri('example.com:path/sp1', SSHRI, hostname='example.com', path='path/sp1')\n    _check_ri('example.com/path/sp1\\\\:fname', PathRI, localpath='example.com/path/sp1\\\\:fname',\n              path='example.com/path/sp1\\\\:fname')\n    # ssh is as stupid as us, so we will stay \"Consistently\" dumb\n    \"\"\"\n    $> ssh example.com/path/sp1:fname\n    ssh: Could not resolve hostname example.com/path/sp1:fname: Name or service not known\n\n    edit 20190516 yoh: but this looks like a perfectly valid path.\n    SSH knows that it is not a path but its SSHRI so it can stay dumb.\n    We are trying to be smart and choose between RIs (even when we know that\n    it is e.g. a file).\n    \"\"\"\n    _check_ri('e.com/p/sp:f', PathRI, localpath='e.com/p/sp:f', path='e.com/p/sp:f')\n    _check_ri('user@someho.st/mydir', PathRI, localpath='user@someho.st/mydir', path='user@someho.st/mydir')\n\n    # SSHRIs have .port, but it is empty\n    eq_(SSHRI(hostname='example.com').port, '')\n\n    # check that we are getting a warning logged when url can't be reconstructed\n    # precisely\n    # actually failed to come up with one -- becomes late here\n    #_check_ri(\"http://host///..//p\", scheme='http', path='/..//p')\n\n    # actually this one is good enough to trigger a warning and I still don't know\n    # what it should exactly be!?\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        weird_str = 'weird://'\n        weird_url = RI(weird_str)\n        repr(weird_url)\n        cml.assert_logged(\n            'Parsed version of SSHRI .weird:/. '\n            'differs from original .weird://.'\n        )\n        # but we store original str\n        eq_(str(weird_url), weird_str)\n        neq_(weird_url.as_str(), weird_str)\n\n    raise SkipTest(\"TODO: file://::1/some does complain about parsed version dropping ::1\")\n\n\ndef test_git_transport_ri():\n    _check_ri(\"gcrypt::http://somewhere\", GitTransportRI, RI='http://somewhere', transport='gcrypt')\n    # man git-push says\n    #  <transport>::<address>\n    #    where <address> may be a path, a server and path, or an arbitrary URL-like string...\n    # so full path to my.com/... should be ok?\n    _check_ri(\"http::/my.com/some/path\", GitTransportRI, RI='/my.com/some/path', transport='http')\n    # some ssh server.  And we allow for some additional chars in transport.\n    # Git doesn't define since it does not care! we will then be flexible too\n    _check_ri(\"trans-port::server:path\", GitTransportRI, RI='server:path', transport='trans-port')\n\n\n@pytest.mark.parametrize(\"cls,clskwargs,target_url\", [\n    (SSHRI, {}, r'example.com:/ \"' + r\"';a&b&cd `| \"),\n    (URL, {'scheme': \"http\"}, 'http://example.com/%20%22%27%3Ba%26b%26cd%20%60%7C%20'),\n    (PathRI, {}, r'/ \"' + r\"';a&b&cd `| \"),  # nothing is done to file:implicit\n])\ndef test_url_quote_path(cls, clskwargs, target_url):\n    path = '/ \"\\';a&b&cd `| '\n    if not (cls is PathRI):\n        clskwargs['hostname'] = hostname = 'example.com'\n    url = cls(path=path, **clskwargs)\n    eq_(url.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n    # all nasty symbols should be quoted\n    url_str = str(url)\n    eq_(url_str, target_url)\n    # no side-effects:\n    eq_(url.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n\n    # and figured out and unquoted\n    url_ = RI(url_str)\n    ok_(isinstance(url_, cls))\n    eq_(url_.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n\n\ndef test_url_compose_archive_one():\n    url = URL(scheme='dl+archive', path='KEY',\n              fragment=OrderedDict((('path', 'f/p/ s+'), ('size', 30))))\n    # funny - space is encoded as + but + is %2B\n    eq_(str(url), 'dl+archive:KEY#path=f/p/+s%2B&size=30')\n    eq_(url.fragment_dict, {'path': 'f/p/ s+', 'size': '30'})\n\n\ndef test_url_fragments_and_query():\n    url = URL(hostname=\"host\", query=OrderedDict((('a', 'x/b'), ('b', 'y'))))\n    eq_(str(url), '//host?a=x%2Fb&b=y')\n    eq_(url.query, 'a=x%2Fb&b=y')\n    eq_(url.query_dict, {'a': 'x/b', 'b': 'y'})\n\n    url = URL(hostname=\"host\", fragment=OrderedDict((('b', 'x/b'), ('a', 'y'))))\n    eq_(str(url), '//host#b=x/b&a=y')\n    eq_(url.fragment, 'b=x/b&a=y')\n    eq_(url.fragment_dict, {'a': 'y', 'b': 'x/b'})\n\n    fname = get_most_obscure_supported_name()\n    url = URL(hostname=\"host\", fragment={'a': fname})\n    eq_(url.fragment_dict, {'a': fname})\n\n\ndef test_url_dicts():\n    eq_(URL(\"http://host\").query_dict, {})\n\n\ndef test_get_url_path_on_fileurls():\n    assert URL('file:///a').path == '/a'\n    assert URL('file:///a/b').path == '/a/b'\n    assert URL('file:///a/b').localpath == local_path_representation('/a/b')\n    assert URL('file:///a/b#id').path == '/a/b'\n    assert URL('file:///a/b?whatever').path == '/a/b'\n\n\ndef test_is_url():\n    ok_(is_url('file://localhost/some'))\n    ok_(is_url('http://localhost'))\n    ok_(is_url('ssh://me@localhost'))\n    # in current understanding it is indeed a url but an 'ssh', implicit=True, not just\n    # a useless scheme=weird with a hope to point to a netloc\n    with swallow_logs():\n        ok_(is_url('weird://'))\n    nok_(is_url('relative'))\n    nok_(is_url('/absolute'))\n    ok_(is_url('like@sshlogin'))  # actually we do allow ssh:implicit urls ATM\n    nok_(is_url(''))\n    nok_(is_url(' '))\n    nok_(is_url(123))  # stuff of other types wouldn't be considered a URL\n\n    # we can pass RI instance directly\n    ok_(is_url(RI('file://localhost/some')))\n    nok_(is_url(RI('relative')))\n\n\n# TODO: RF with test_is_url to avoid duplication\ndef test_is_datalad_compat_ri():\n    ok_(is_datalad_compat_ri('ssh://user:passw@host/path'))\n    ok_(is_datalad_compat_ri('http://example.com'))\n    ok_(is_datalad_compat_ri('file://localhost/some'))\n    ok_(is_datalad_compat_ri('///localhost/some'))\n    nok_(is_datalad_compat_ri('relative'))\n    nok_(is_datalad_compat_ri('.///localhost/some'))\n    nok_(is_datalad_compat_ri(123))\n\n\ndef test_get_local_file_url():\n    compat_annex = 'git-annex'\n    compat_git = 'git'\n    for path, url, compatibility in (\n                # relpaths are special-cased below\n                ('test.txt', 'test.txt', compat_annex),\n                (OBSCURE_FILENAME, urlquote(OBSCURE_FILENAME), compat_annex),\n            ) + ((\n                ('C:\\\\Windows\\\\notepad.exe', 'file://C:/Windows/notepad.exe', compat_annex),\n                ('C:\\\\Windows\\\\notepad.exe', 'file:///C:/Windows/notepad.exe', compat_git),\n            ) if on_windows else (\n                ('/a', 'file:///a', compat_annex),\n                ('/a/b/c', 'file:///a/b/c', compat_annex),\n                ('/a~', 'file:///a~', compat_annex),\n                # there are no files with trailing slashes in the name\n                #('/a b/', 'file:///a%20b/'),\n                ('/a b/name', 'file:///a%20b/name', compat_annex),\n            )):\n        # Yarik found no better way to trigger.  .decode() isn't enough\n        print(\"D: %s\" % path)\n        if isabs(path):\n            assert get_local_file_url(path, compatibility=compatibility) == url\n            abs_path = path\n        else:\n            assert get_local_file_url(path, allow_relative_path=True, compatibility=compatibility) \\\n                   == '/'.join((get_local_file_url(os.getcwd(), compatibility=compatibility), url))\n            abs_path = opj(os.getcwd(), path)\n        if compatibility == compat_git:\n            assert get_local_file_url(abs_path, compatibility=compatibility) == Path(abs_path).as_uri()\n\n\n@with_tempfile(mkdir=True)\ndef test_get_local_file_url_compatibility(path=None):\n    # smoke test for file:// URL compatibility with other datalad/git/annex\n    # pieces\n    path = Path(path)\n    ds1 = Dataset(path / 'ds1').create()\n    ds2 = Dataset(path / 'ds2').create()\n    testfile = path / 'testfile.txt'\n    testfile.write_text('some')\n\n    # compat with annex addurl\n    ds1.repo.add_url_to_file(\n        'test.txt',\n        get_local_file_url(str(testfile), compatibility='git-annex'))\n\n    # compat with git clone/submodule\n    assert_status(\n        'ok',\n        ds1.clone(get_local_file_url(ds2.path, compatibility='git'),\n                  result_xfm=None, return_type='generator'))\n\n\ndef test_is_ssh():\n\n    ssh_locators = [\"ssh://host\",\n                    \"ssh://host/some/where\",\n                    \"user@host:path/sp1\",\n                    \"user@host:/absolute/path/sp1\",\n                    \"host:path/sp1\",\n                    \"host:/absolute/path/sp1\",\n                    \"user@host\"]\n    for ri in ssh_locators:\n        ok_(is_ssh(ri), \"not considered ssh (string): %s\" % ri)\n        ok_(is_ssh(RI(ri)), \"not considered ssh (RI): %s\" % ri)\n\n    non_ssh_locators = [\"file://path/to\",\n                        \"/abs/path\",\n                        \"../rel/path\",\n                        \"http://example.com\",\n                        \"git://host/user/proj\",\n                        \"s3://bucket/save/?key=891\"]\n    for ri in non_ssh_locators:\n        ok_(not is_ssh(ri), \"considered ssh (string): %s\" % ri)\n        ok_(not is_ssh(RI(ri)), \"considered ssh (RI): %s\" % ri)\n\n\ndef test_iso8601_to_epoch():\n    epoch = 1467901515\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15+00:00'), epoch)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15+11:00'),\n        epoch - 11 * 60 * 60)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15Z'), epoch)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15'), epoch)\n\n    eq_(iso8601_to_epoch('2016-07-07T14:25:14'), epoch-1)\n\n\ndef test_mapping_identity():\n    from datalad.tests.utils_pytest import OBSCURE_FILENAME\n\n    absolute_obscure_path = str(Path('/').absolute() / OBSCURE_FILENAME)\n    temp_dir = tempfile.gettempdir()\n    print(f\"temp_dir: {temp_dir}\")\n    for name in (temp_dir, opj(temp_dir, \"x.txt\"), absolute_obscure_path):\n        # On some platforms, e.g. MacOS, `temp_dir` might contain trailing\n        # slashes. Since the conversion and its inverse normalize paths, we\n        # compare the result to the normalized path\n        normalized_name = str(Path(name))\n        assert url_path2local_path(local_path2url_path(name)) == normalized_name\n\n    prefix = \"/C:\" if on_windows else \"\"\n    for name in map(quote_path, (prefix + \"/window\", prefix + \"/d\", prefix + \"/\" + OBSCURE_FILENAME)):\n        assert local_path2url_path(url_path2local_path(name)) == name\n\n\ndef test_auto_resolve_path():\n    relative_path = str(Path(\"a/b\"))\n    with pytest.raises(ValueError):\n        local_path2url_path(relative_path)\n    local_path2url_path(\"\", allow_relative_path=True)\n\n\n@skip_if(not on_windows)\ndef test_hostname_detection():\n    with pytest.raises(ValueError):\n        local_path2url_path(\"\\\\\\\\server\\\\share\\\\path\")\n\n\ndef test_url_path2local_path_excceptions():\n    with pytest.raises(ValueError):\n        url_path2local_path('')\n    with pytest.raises(ValueError):\n        url_path2local_path(None)\n    with pytest.raises(ValueError):\n        url_path2local_path('a/b')\n    with pytest.raises(ValueError):\n        url_path2local_path(PurePosixPath('a/b'))\n    with pytest.raises(ValueError):\n        url_path2local_path(PurePosixPath('//a/b'))\n\n\ndef test_quote_path(monkeypatch):\n    with monkeypatch.context() as ctx:\n        ctx.setattr(datalad.support.network, 'on_windows', True)\n        assert quote_path(\"/c:/win:xxx\") == \"/c:/win%3Axxx\"\n        assert quote_path(\"/C:/win:xxx\") == \"/C:/win%3Axxx\"\n\n        ctx.setattr(datalad.support.network, 'on_windows', False)\n        assert quote_path(\"/c:/win:xxx\") == \"/c%3A/win%3Axxx\"\n        assert quote_path(\"/C:/win:xxx\") == \"/C%3A/win%3Axxx\"\n",
            "header": {
                "file": {
                    "path": "datalad/support/tests/test_network.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "dfdaa820b",
                "new_commit_hash": "356699b51",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/tests/test_network.py"
            },
            "plus_file": {
                "path": "b/datalad/support/tests/test_network.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 508,
                            "length": 7
                        },
                        "new_range": {
                            "start": 508,
                            "length": 7
                        },
                        "section": "def test_get_local_file_url():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                # relpaths are special-cased below",
                                "type": "context"
                            },
                            {
                                "content": "                ('test.txt', 'test.txt', compat_annex),",
                                "type": "context"
                            },
                            {
                                "content": "                (OBSCURE_FILENAME, urlquote(OBSCURE_FILENAME), compat_annex),",
                                "type": "context"
                            },
                            {
                                "content": "            ) + (",
                                "type": "deleted"
                            },
                            {
                                "content": "            ) + ((",
                                "type": "added"
                            },
                            {
                                "content": "                ('C:\\\\Windows\\\\notepad.exe', 'file://C:/Windows/notepad.exe', compat_annex),",
                                "type": "context"
                            },
                            {
                                "content": "                ('C:\\\\Windows\\\\notepad.exe', 'file:///C:/Windows/notepad.exe', compat_git),",
                                "type": "context"
                            },
                            {
                                "content": "            ) if on_windows else (",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/tests/test_network.py",
                            "type": "function",
                            "name": "test_get_local_file_url",
                            "content": "def test_get_local_file_url():\n    compat_annex = 'git-annex'\n    compat_git = 'git'\n    for path, url, compatibility in (\n                # relpaths are special-cased below\n                ('test.txt', 'test.txt', compat_annex),\n                (OBSCURE_FILENAME, urlquote(OBSCURE_FILENAME), compat_annex),\n            ) + (\n                ('C:\\\\Windows\\\\notepad.exe', 'file://C:/Windows/notepad.exe', compat_annex),\n                ('C:\\\\Windows\\\\notepad.exe', 'file:///C:/Windows/notepad.exe', compat_git),\n            ) if on_windows else (\n                ('/a', 'file:///a', compat_annex),\n                ('/a/b/c', 'file:///a/b/c', compat_annex),\n                ('/a~', 'file:///a~', compat_annex),\n                # there are no files with trailing slashes in the name\n                #('/a b/', 'file:///a%20b/'),\n                ('/a b/name', 'file:///a%20b/name', compat_annex),\n            ):\n        # Yarik found no better way to trigger.  .decode() isn't enough\n        print(\"D: %s\" % path)\n        if isabs(path):\n            assert get_local_file_url(path, compatibility=compatibility) == url\n        else:\n            assert get_local_file_url(path, allow_relative_path=True, compatibility=compatibility) \\\n                   == '/'.join((get_local_file_url(os.getcwd(), compatibility=compatibility), url))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 504,
                            "end_lineno": 528,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 518,
                            "length": 14
                        },
                        "new_range": {
                            "start": 518,
                            "length": 19
                        },
                        "section": "def test_get_local_file_url():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                # there are no files with trailing slashes in the name",
                                "type": "context"
                            },
                            {
                                "content": "                #('/a b/', 'file:///a%20b/'),",
                                "type": "context"
                            },
                            {
                                "content": "                ('/a b/name', 'file:///a%20b/name', compat_annex),",
                                "type": "context"
                            },
                            {
                                "content": "            ):",
                                "type": "deleted"
                            },
                            {
                                "content": "            )):",
                                "type": "added"
                            },
                            {
                                "content": "        # Yarik found no better way to trigger.  .decode() isn't enough",
                                "type": "context"
                            },
                            {
                                "content": "        print(\"D: %s\" % path)",
                                "type": "context"
                            },
                            {
                                "content": "        if isabs(path):",
                                "type": "context"
                            },
                            {
                                "content": "            assert get_local_file_url(path, compatibility=compatibility) == url",
                                "type": "context"
                            },
                            {
                                "content": "            abs_path = path",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "context"
                            },
                            {
                                "content": "            assert get_local_file_url(path, allow_relative_path=True, compatibility=compatibility) \\",
                                "type": "context"
                            },
                            {
                                "content": "                   == '/'.join((get_local_file_url(os.getcwd(), compatibility=compatibility), url))",
                                "type": "context"
                            },
                            {
                                "content": "            abs_path = opj(os.getcwd(), path)",
                                "type": "added"
                            },
                            {
                                "content": "        if compatibility == compat_git:",
                                "type": "added"
                            },
                            {
                                "content": "            assert get_local_file_url(abs_path, compatibility=compatibility) == Path(abs_path).as_uri()",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@with_tempfile(mkdir=True)",
                                "type": "context"
                            },
                            {
                                "content": "def test_get_local_file_url_compatibility(path=None):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/tests/test_network.py",
                            "type": "function",
                            "name": "test_get_local_file_url",
                            "content": "def test_get_local_file_url():\n    compat_annex = 'git-annex'\n    compat_git = 'git'\n    for path, url, compatibility in (\n                # relpaths are special-cased below\n                ('test.txt', 'test.txt', compat_annex),\n                (OBSCURE_FILENAME, urlquote(OBSCURE_FILENAME), compat_annex),\n            ) + (\n                ('C:\\\\Windows\\\\notepad.exe', 'file://C:/Windows/notepad.exe', compat_annex),\n                ('C:\\\\Windows\\\\notepad.exe', 'file:///C:/Windows/notepad.exe', compat_git),\n            ) if on_windows else (\n                ('/a', 'file:///a', compat_annex),\n                ('/a/b/c', 'file:///a/b/c', compat_annex),\n                ('/a~', 'file:///a~', compat_annex),\n                # there are no files with trailing slashes in the name\n                #('/a b/', 'file:///a%20b/'),\n                ('/a b/name', 'file:///a%20b/name', compat_annex),\n            ):\n        # Yarik found no better way to trigger.  .decode() isn't enough\n        print(\"D: %s\" % path)\n        if isabs(path):\n            assert get_local_file_url(path, compatibility=compatibility) == url\n        else:\n            assert get_local_file_url(path, allow_relative_path=True, compatibility=compatibility) \\\n                   == '/'.join((get_local_file_url(os.getcwd(), compatibility=compatibility), url))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 504,
                            "end_lineno": 528,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 606,
                            "length": 7
                        },
                        "new_range": {
                            "start": 611,
                            "length": 7
                        },
                        "section": "def test_auto_resolve_path():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    relative_path = str(Path(\"a/b\"))",
                                "type": "context"
                            },
                            {
                                "content": "    with pytest.raises(ValueError):",
                                "type": "context"
                            },
                            {
                                "content": "        local_path2url_path(relative_path)",
                                "type": "context"
                            },
                            {
                                "content": "    local_path2url_path(\"\", auto_resolve=True)",
                                "type": "deleted"
                            },
                            {
                                "content": "    local_path2url_path(\"\", allow_relative_path=True)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@skip_if(not on_windows)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/tests/test_network.py",
                            "type": "function",
                            "name": "test_auto_resolve_path",
                            "content": "def test_auto_resolve_path():\n    relative_path = str(Path(\"a/b\"))\n    with pytest.raises(ValueError):\n        local_path2url_path(relative_path)\n    local_path2url_path(\"\", auto_resolve=True)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 605,
                            "end_lineno": 609,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "4c614de1c1e352dddbd135961c0ae4595eb3053f^",
    "new_commit_hash": "4c614de1c1e352dddbd135961c0ae4595eb3053f",
    "commit_message": "clean up parameter of path conversion functions\n\nThis commit removes the unused parameter `make_absolute`\nfrom `url_path2local_path`. It also renames the\nparamter `auto_resolve` of function `local_path2url_path`\nto `allow_relative_path` and delegates relative\npath handling from `get_local_file_url' to\n`local_path2url_path`.\n\nAlso moved the runtime comparison of\n`get_local_file_url` the result of\n`Path.as_uri()` into tests.",
    "commit_date": "2022-12-21T16:59:00+01:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "4c614de1c1e352dddbd135961c0ae4595eb3053f",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 et:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport logging\nimport os\nimport tempfile\nfrom collections import OrderedDict\nfrom os.path import isabs\nfrom os.path import join as opj\n\nimport pytest\n\nimport datalad.support.network\nfrom datalad.distribution.dataset import Dataset\nfrom datalad.support.network import (\n    RI,\n    SSHRI,\n    URL,\n    DataLadRI,\n    GitTransportRI,\n    PathRI,\n    _split_colon,\n    dlurljoin,\n    get_local_file_url,\n    get_response_disposition_filename,\n    get_tld,\n    get_url_straight_filename,\n    is_datalad_compat_ri,\n    is_ssh,\n    is_url,\n    iso8601_to_epoch,\n    local_path_representation,\n    local_path2url_path,\n    local_url_path_representation,\n    parse_url_opts,\n    quote_path,\n    same_website,\n    urlquote,\n    url_path2local_path,\n)\nfrom datalad.tests.utils_pytest import (\n    OBSCURE_FILENAME,\n    SkipTest,\n    assert_in,\n    assert_raises,\n    assert_status,\n    eq_,\n    get_most_obscure_supported_name,\n    known_failure_githubci_win,\n    neq_,\n    nok_,\n    ok_,\n    skip_if,\n    swallow_logs,\n    with_tempfile,\n)\nfrom datalad.utils import (\n    Path,\n    PurePosixPath,\n    on_windows,\n)\n\n\ndef test_same_website():\n    ok_(same_website(\"http://a.b\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"https://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"https://a.b/2014/01/xxx/\"))\n\n\ndef test_get_tld():\n    eq_(get_tld('http://example.com'), 'example.com')\n    eq_(get_tld('http://example.com/1'), 'example.com')\n    eq_(get_tld('http://example.com/1/2'), 'example.com')\n    eq_(get_tld('example.com/1/2'), 'example.com')\n    eq_(get_tld('s3://example.com/1/2'), 'example.com')\n    assert_raises(ValueError, get_tld, \"\")\n    assert_raises(ValueError, get_tld, \"s3://\")\n    assert_raises(ValueError, get_tld, \"http://\")\n\ndef test_dlurljoin():\n    eq_(dlurljoin('http://a.b/', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/page', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/dir/', 'f'), 'http://a.b/dir/f')\n    eq_(dlurljoin('http://a.b/dir/', 'http://url'), 'http://url')\n    eq_(dlurljoin('http://a.b/dir/', '/'), 'http://a.b/')\n    eq_(dlurljoin('http://a.b/dir/', '/x/y'), 'http://a.b/x/y')\n\n@pytest.mark.parametrize(\"suf\", [\n    '',\n    '#',\n    '#tag',\n    '#tag/obscure',\n    '?param=1',\n    '?param=1&another=/',\n])\ndef test_get_url_straight_filename(suf):\n    eq_(get_url_straight_filename('http://a.b/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')\n\nfrom datalad.support.network import rfc2822_to_epoch\n\n\ndef test_rfc2822_to_epoch():\n    eq_(rfc2822_to_epoch(\"Thu, 16 Oct 2014 01:16:17 EDT\"), 1413436577)\n\n\ndef test_get_response_disposition_filename():\n    eq_(get_response_disposition_filename('attachment;filename=\"Part1-Subjects1-99.tar\"'), \"Part1-Subjects1-99.tar\")\n    eq_(get_response_disposition_filename('attachment'), None)\n\n\ndef test_parse_url_opts():\n    url = 'http://map.org/api/download/?id=157'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '157'}))\n\n    url = 's3://bucket/save/?key=891'\n    output = parse_url_opts(url)\n    eq_(output, ('s3://bucket/save/', {'key': '891'}))\n\n    url = 'http://map.org/api/download/?id=98&code=13'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '98', 'code': '13'}))\n\n\ndef test_split_colon():\n    eq_(_split_colon('a:b'), ['a', 'b'])\n    eq_(_split_colon('a:b:c'), ['a', 'b:c'])\n    eq_(_split_colon('a:b:c', 2), ['a', 'b', 'c'])\n    eq_(_split_colon('ab'), ['ab'])\n    eq_(_split_colon(r'a\\:b'), [r'a\\:b'])\n\n\ndef test_url_eq():\n    assert URL() == URL()\n    # doesn't make sense to ask what kind of a url it is an empty URL\n    assert URL() != URL(hostname='x')\n    # Different types aren't equal even if have the same fields values\n    assert URL(path='x') != PathRI(path='x')\n    assert URL(hostname='x') != SSHRI(hostname='x')\n    assert str(URL(hostname='x')) != str(SSHRI(hostname='x'))\n\n\ndef _check_ri(ri, cls, exact_str=True, localpath=None, **fields):\n    \"\"\"just a helper to carry out few checks on urls\"\"\"\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ri_ = cls(**fields)\n        murl = RI(ri)\n        assert murl.__class__ == cls  # not just a subclass\n        assert murl == ri_\n        if isinstance(ri, str):\n            assert str(RI(ri)) == ri\n        assert eval(repr(ri_)) == ri  # repr leads back to identical ri_\n        assert ri == ri_  # just in case ;)  above should fail first if smth is wrong\n        if not exact_str:\n            assert_in('Parsed version of', cml.out)\n    if exact_str:\n        assert str(ri) == str(ri_)\n    else:\n        assert str(ri) != str(ri_)\n\n    # and that we have access to all those fields\n    nok_(set(fields).difference(set(cls._FIELDS)))\n    for f, v in fields.items():\n        assert getattr(ri_, f) == v\n\n    if localpath:\n        if cls == URL:\n            local_representation = local_url_path_representation(localpath)\n        else:\n            local_representation = local_path_representation(localpath)\n        assert ri_.localpath == local_representation\n        old_localpath = ri_.localpath  # for a test below\n    else:\n        # if not given -- must be a remote url, should raise exception on\n        # non-Windows systems. But not on Windows systems because we allow UNCs\n        # to be encoded in URLs\n        if not on_windows:\n            with assert_raises(ValueError):\n                ri_.localpath\n\n    # This one does not have a path. TODO: either proxy path from its .RI or adjust\n    # hierarchy of classes to make it more explicit\n    if cls == GitTransportRI:\n        return\n    # do changes in the path persist?\n    old_str = str(ri_)\n    ri_.path = newpath = opj(ri_.path, 'sub')\n    assert ri_.path == newpath\n    assert str(ri_) != old_str\n    if localpath:\n        assert ri_.localpath == local_path_representation(opj(old_localpath, 'sub'))\n\n\ndef test_url_base():\n    # Basic checks\n    assert_raises(ValueError, URL, \"http://example.com\", hostname='example.com')\n    url = URL(\"http://example.com\")\n    eq_(url.hostname, 'example.com')\n    eq_(url.scheme, 'http')\n    eq_(url.port, '')  # not specified -- empty strings\n    eq_(url.username, '')  # not specified -- empty strings\n    eq_(repr(url), \"URL(hostname='example.com', netloc='example.com', scheme='http')\")\n    eq_(url, \"http://example.com\")  # automagic coercion in __eq__\n\n    neq_(URL(), URL(hostname='x'))\n\n    smth = URL('smth')\n    eq_(smth.hostname, '')\n    ok_(bool(smth))\n    nok_(bool(URL()))\n\n    assert_raises(ValueError, url._set_from_fields, unknown='1')\n\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        # we don't \"care\" about params ATM so there is a warning if there are any\n        purl = URL(\"http://example.com/;param\")\n        eq_(str(purl), 'http://example.com/;param')  # but we do maintain original string\n        assert_in('ParseResults contains params', cml.out)\n        eq_(purl.as_str(), 'http://example.com/')\n\n\n@with_tempfile\ndef test_pathri_guessing(filename=None):\n    # Complaining about ;param only at DEBUG level\n    # see https://github.com/datalad/datalad/issues/6872\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        # we don't \"care\" about params ATM so there is a warning if there are any\n        ri = RI(f\"{filename};param\")\n        assert isinstance(ri, PathRI)\n        if not on_windows:\n            # Does not happen on Windows since paths with \\ instead of / do not\n            # look like possible URLs\n            assert_in('ParseResults contains params', cml.out)\n\n\n@skip_if(not on_windows)\ndef test_pathri_windows_anchor():\n    assert RI('file:///c:/Windows').localpath == 'C:\\\\Windows'\n\n\n@known_failure_githubci_win\ndef test_url_samples():\n    _check_ri(\"http://example.com\", URL, scheme='http', hostname=\"example.com\", netloc='example.com')\n    # \"complete\" one for classical http\n    _check_ri(\"http://user:pw@example.com:8080/p/sp?p1=v1&p2=v2#frag\", URL,\n              scheme='http', netloc='user:pw@example.com:8080',\n              hostname=\"example.com\", port=8080, username='user', password='pw',\n              path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n\n    # sample one for ssh with specifying the scheme\n    # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n    _check_ri(\"ssh://host/path/sp1\", URL, scheme='ssh', hostname='host',\n              netloc='host', path='/path/sp1')\n    _check_ri(\"user@host:path/sp1\", SSHRI,\n              hostname='host', path='path/sp1', username='user')\n    _check_ri(\"host:path/sp1\", SSHRI, hostname='host', path='path/sp1')\n    _check_ri(\"host:path\", SSHRI, hostname='host', path='path')\n    _check_ri(\"host:/path\", SSHRI, hostname='host', path='/path')\n    _check_ri(\"user@host\", SSHRI, hostname='host', username='user')\n    # TODO!!!  should this be a legit URL like this?\n    # _check_ri(\"host\", SSHRI, hostname='host'))\n    eq_(repr(RI(\"host:path\")), \"SSHRI(hostname='host', path='path')\")\n\n    # And now perspective 'datalad', implicit=True urls pointing to the canonical center location\n    _check_ri(\"///\", DataLadRI)\n    _check_ri(\"///p/s1\", DataLadRI, path='p/s1')\n    # could be considered by someone as \"URI reference\" relative to scheme\n    _check_ri(\"//a/\", DataLadRI, remote='a')\n    _check_ri(\"//a/data\", DataLadRI, path='data', remote='a')\n\n    # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n    # or not so custom as\n    _check_ri(\"hg+https://host/user/proj\", URL, scheme=\"hg+https\",\n              netloc='host', hostname='host', path='/user/proj')\n    # \"old\" style\n    _check_ri(\"dl+archive:KEY/path/sp1#size=123\", URL,\n              scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n    # \"new\" style\n    _check_ri(\"dl+archive:KEY#path=path/sp1&size=123\", URL,\n              scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n    # actually above one is probably wrong since we need to encode the path\n    _check_ri(\"dl+archive:KEY#path=path%2Fbsp1&size=123\", URL,\n              scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n\n    #https://en.wikipedia.org/wiki/File_URI_scheme\n    _check_ri(\"file://host\", URL, scheme='file', netloc='host', hostname='host')\n    _check_ri(\"file://host/path/sp1\", URL, scheme='file', netloc='host',\n              hostname='host', path='/path/sp1')\n    # stock libraries of Python aren't quite ready for ipv6\n    ipv6address = '2001:db8:85a3::8a2e:370:7334'\n    _check_ri(\"file://%s/path/sp1\" % ipv6address, URL,\n              scheme='file', netloc=ipv6address, hostname=ipv6address,\n              path='/path/sp1')\n    for lh in ('localhost', '::1', '', '127.3.4.155'):\n        if on_windows:\n            url = RI(f\"file://{lh}/path/sp1\")\n            assert url.localpath == f'\\\\\\\\{lh}\\\\path\\\\sp1' if lh else '\\\\path\\\\sp1'\n        else:\n            _check_ri(\"file://%s/path/sp1\" % lh, URL, localpath='/path/sp1',\n                      scheme='file', netloc=lh, hostname=lh, path='/path/sp1')\n\n    _check_ri('http://[1fff:0:a88:85a3::ac1f]:8001/index.html', URL,\n              scheme='http', netloc='[1fff:0:a88:85a3::ac1f]:8001',\n              hostname='1fff:0:a88:85a3::ac1f', port=8001, path='/index.html')\n    _check_ri(\"file:///path/sp1\", URL, localpath='/path/sp1', scheme='file', path='/path/sp1')\n    # we don't do any magical comprehension for home paths/drives for windows\n    # of file:// urls, thus leaving /~ and /c: for now:\n    _check_ri(\"file:///~/path/sp1\", URL, localpath='/~/path/sp1', scheme='file', path='/~/path/sp1')\n    _check_ri(\"file:///%7E/path/sp1\", URL, localpath='/~/path/sp1', scheme='file', path='/~/path/sp1', exact_str=False)\n    # not sure but let's check\n    if on_windows:\n        _check_ri(\"file:///C:/path/sp1\", URL, localpath='C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        _check_ri(\"file:/C:/path/sp1\", URL, localpath='C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        # git-annex style drive-letter encoding\n        _check_ri(\"file://C:/path/sp1\", URL, netloc=\"C:\", hostname=\"c\", localpath='C:/path/sp1', scheme='file', path='/path/sp1', exact_str=False)\n    else:\n        _check_ri(\"file:///C:/path/sp1\", URL, localpath='/C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n        _check_ri(\"file:/C:/path/sp1\", URL, localpath='/C:/path/sp1', scheme='file', path='/C:/path/sp1', exact_str=False)\n\n    # and now implicit paths or actually they are also \"URI references\"\n    _check_ri(\"f\", PathRI, localpath='f', path='f')\n    _check_ri(\"f/s1\", PathRI, localpath='f/s1', path='f/s1')\n    _check_ri(PurePosixPath(\"f\"), PathRI, localpath='f', path='f')\n    _check_ri(PurePosixPath(\"f/s1\"), PathRI, localpath='f/s1', path='f/s1')\n    # colons are problematic and might cause confusion into SSHRI\n    _check_ri(\"f/s:1\", PathRI, localpath='f/s:1', path='f/s:1')\n    _check_ri(\"f/s:\", PathRI, localpath='f/s:', path='f/s:')\n    _check_ri(\"/f\", PathRI, localpath='/f', path='/f')\n    _check_ri(\"/f/s1\", PathRI, localpath='/f/s1', path='/f/s1')\n\n    # some github ones, just to make sure\n    _check_ri(\"git://host/user/proj\", URL, scheme=\"git\", netloc=\"host\",\n              hostname=\"host\", path=\"/user/proj\")\n    _check_ri(\"git@host:user/proj\", SSHRI, hostname=\"host\", path=\"user/proj\", username='git')\n\n    _check_ri('weird:/', SSHRI, hostname='weird', path='/')\n    # since schema is not allowing some symbols so we need to add additional check\n    _check_ri('weird_url:/', SSHRI, hostname='weird_url', path='/')\n    _check_ri('example.com:/', SSHRI, hostname='example.com', path='/')\n    _check_ri('example.com:path/sp1', SSHRI, hostname='example.com', path='path/sp1')\n    _check_ri('example.com/path/sp1\\\\:fname', PathRI, localpath='example.com/path/sp1\\\\:fname',\n              path='example.com/path/sp1\\\\:fname')\n    # ssh is as stupid as us, so we will stay \"Consistently\" dumb\n    \"\"\"\n    $> ssh example.com/path/sp1:fname\n    ssh: Could not resolve hostname example.com/path/sp1:fname: Name or service not known\n\n    edit 20190516 yoh: but this looks like a perfectly valid path.\n    SSH knows that it is not a path but its SSHRI so it can stay dumb.\n    We are trying to be smart and choose between RIs (even when we know that\n    it is e.g. a file).\n    \"\"\"\n    _check_ri('e.com/p/sp:f', PathRI, localpath='e.com/p/sp:f', path='e.com/p/sp:f')\n    _check_ri('user@someho.st/mydir', PathRI, localpath='user@someho.st/mydir', path='user@someho.st/mydir')\n\n    # SSHRIs have .port, but it is empty\n    eq_(SSHRI(hostname='example.com').port, '')\n\n    # check that we are getting a warning logged when url can't be reconstructed\n    # precisely\n    # actually failed to come up with one -- becomes late here\n    #_check_ri(\"http://host///..//p\", scheme='http', path='/..//p')\n\n    # actually this one is good enough to trigger a warning and I still don't know\n    # what it should exactly be!?\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        weird_str = 'weird://'\n        weird_url = RI(weird_str)\n        repr(weird_url)\n        cml.assert_logged(\n            'Parsed version of SSHRI .weird:/. '\n            'differs from original .weird://.'\n        )\n        # but we store original str\n        eq_(str(weird_url), weird_str)\n        neq_(weird_url.as_str(), weird_str)\n\n    raise SkipTest(\"TODO: file://::1/some does complain about parsed version dropping ::1\")\n\n\ndef test_git_transport_ri():\n    _check_ri(\"gcrypt::http://somewhere\", GitTransportRI, RI='http://somewhere', transport='gcrypt')\n    # man git-push says\n    #  <transport>::<address>\n    #    where <address> may be a path, a server and path, or an arbitrary URL-like string...\n    # so full path to my.com/... should be ok?\n    _check_ri(\"http::/my.com/some/path\", GitTransportRI, RI='/my.com/some/path', transport='http')\n    # some ssh server.  And we allow for some additional chars in transport.\n    # Git doesn't define since it does not care! we will then be flexible too\n    _check_ri(\"trans-port::server:path\", GitTransportRI, RI='server:path', transport='trans-port')\n\n\n@pytest.mark.parametrize(\"cls,clskwargs,target_url\", [\n    (SSHRI, {}, r'example.com:/ \"' + r\"';a&b&cd `| \"),\n    (URL, {'scheme': \"http\"}, 'http://example.com/%20%22%27%3Ba%26b%26cd%20%60%7C%20'),\n    (PathRI, {}, r'/ \"' + r\"';a&b&cd `| \"),  # nothing is done to file:implicit\n])\ndef test_url_quote_path(cls, clskwargs, target_url):\n    path = '/ \"\\';a&b&cd `| '\n    if not (cls is PathRI):\n        clskwargs['hostname'] = hostname = 'example.com'\n    url = cls(path=path, **clskwargs)\n    eq_(url.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n    # all nasty symbols should be quoted\n    url_str = str(url)\n    eq_(url_str, target_url)\n    # no side-effects:\n    eq_(url.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n\n    # and figured out and unquoted\n    url_ = RI(url_str)\n    ok_(isinstance(url_, cls))\n    eq_(url_.path, path)\n    if 'hostname' in clskwargs:\n        eq_(url.hostname, hostname)\n\n\ndef test_url_compose_archive_one():\n    url = URL(scheme='dl+archive', path='KEY',\n              fragment=OrderedDict((('path', 'f/p/ s+'), ('size', 30))))\n    # funny - space is encoded as + but + is %2B\n    eq_(str(url), 'dl+archive:KEY#path=f/p/+s%2B&size=30')\n    eq_(url.fragment_dict, {'path': 'f/p/ s+', 'size': '30'})\n\n\ndef test_url_fragments_and_query():\n    url = URL(hostname=\"host\", query=OrderedDict((('a', 'x/b'), ('b', 'y'))))\n    eq_(str(url), '//host?a=x%2Fb&b=y')\n    eq_(url.query, 'a=x%2Fb&b=y')\n    eq_(url.query_dict, {'a': 'x/b', 'b': 'y'})\n\n    url = URL(hostname=\"host\", fragment=OrderedDict((('b', 'x/b'), ('a', 'y'))))\n    eq_(str(url), '//host#b=x/b&a=y')\n    eq_(url.fragment, 'b=x/b&a=y')\n    eq_(url.fragment_dict, {'a': 'y', 'b': 'x/b'})\n\n    fname = get_most_obscure_supported_name()\n    url = URL(hostname=\"host\", fragment={'a': fname})\n    eq_(url.fragment_dict, {'a': fname})\n\n\ndef test_url_dicts():\n    eq_(URL(\"http://host\").query_dict, {})\n\n\ndef test_get_url_path_on_fileurls():\n    assert URL('file:///a').path == '/a'\n    assert URL('file:///a/b').path == '/a/b'\n    assert URL('file:///a/b').localpath == local_path_representation('/a/b')\n    assert URL('file:///a/b#id').path == '/a/b'\n    assert URL('file:///a/b?whatever').path == '/a/b'\n\n\ndef test_is_url():\n    ok_(is_url('file://localhost/some'))\n    ok_(is_url('http://localhost'))\n    ok_(is_url('ssh://me@localhost'))\n    # in current understanding it is indeed a url but an 'ssh', implicit=True, not just\n    # a useless scheme=weird with a hope to point to a netloc\n    with swallow_logs():\n        ok_(is_url('weird://'))\n    nok_(is_url('relative'))\n    nok_(is_url('/absolute'))\n    ok_(is_url('like@sshlogin'))  # actually we do allow ssh:implicit urls ATM\n    nok_(is_url(''))\n    nok_(is_url(' '))\n    nok_(is_url(123))  # stuff of other types wouldn't be considered a URL\n\n    # we can pass RI instance directly\n    ok_(is_url(RI('file://localhost/some')))\n    nok_(is_url(RI('relative')))\n\n\n# TODO: RF with test_is_url to avoid duplication\ndef test_is_datalad_compat_ri():\n    ok_(is_datalad_compat_ri('ssh://user:passw@host/path'))\n    ok_(is_datalad_compat_ri('http://example.com'))\n    ok_(is_datalad_compat_ri('file://localhost/some'))\n    ok_(is_datalad_compat_ri('///localhost/some'))\n    nok_(is_datalad_compat_ri('relative'))\n    nok_(is_datalad_compat_ri('.///localhost/some'))\n    nok_(is_datalad_compat_ri(123))\n\n\ndef test_get_local_file_url():\n    compat_annex = 'git-annex'\n    compat_git = 'git'\n    for path, url, compatibility in (\n                # relpaths are special-cased below\n                ('test.txt', 'test.txt', compat_annex),\n                (OBSCURE_FILENAME, urlquote(OBSCURE_FILENAME), compat_annex),\n            ) + ((\n                ('C:\\\\Windows\\\\notepad.exe', 'file://C:/Windows/notepad.exe', compat_annex),\n                ('C:\\\\Windows\\\\notepad.exe', 'file:///C:/Windows/notepad.exe', compat_git),\n            ) if on_windows else (\n                ('/a', 'file:///a', compat_annex),\n                ('/a/b/c', 'file:///a/b/c', compat_annex),\n                ('/a~', 'file:///a~', compat_annex),\n                # there are no files with trailing slashes in the name\n                #('/a b/', 'file:///a%20b/'),\n                ('/a b/name', 'file:///a%20b/name', compat_annex),\n            )):\n        # Yarik found no better way to trigger.  .decode() isn't enough\n        print(\"D: %s\" % path)\n        if isabs(path):\n            assert get_local_file_url(path, compatibility=compatibility) == url\n            abs_path = path\n        else:\n            assert get_local_file_url(path, allow_relative_path=True, compatibility=compatibility) \\\n                   == '/'.join((get_local_file_url(os.getcwd(), compatibility=compatibility), url))\n            abs_path = opj(os.getcwd(), path)\n        if compatibility == compat_git:\n            assert get_local_file_url(abs_path, compatibility=compatibility) == Path(abs_path).as_uri()\n\n\n@with_tempfile(mkdir=True)\ndef test_get_local_file_url_compatibility(path=None):\n    # smoke test for file:// URL compatibility with other datalad/git/annex\n    # pieces\n    path = Path(path)\n    ds1 = Dataset(path / 'ds1').create()\n    ds2 = Dataset(path / 'ds2').create()\n    testfile = path / 'testfile.txt'\n    testfile.write_text('some')\n\n    # compat with annex addurl\n    ds1.repo.add_url_to_file(\n        'test.txt',\n        get_local_file_url(str(testfile), compatibility='git-annex'))\n\n    # compat with git clone/submodule\n    assert_status(\n        'ok',\n        ds1.clone(get_local_file_url(ds2.path, compatibility='git'),\n                  result_xfm=None, return_type='generator'))\n\n\ndef test_is_ssh():\n\n    ssh_locators = [\"ssh://host\",\n                    \"ssh://host/some/where\",\n                    \"user@host:path/sp1\",\n                    \"user@host:/absolute/path/sp1\",\n                    \"host:path/sp1\",\n                    \"host:/absolute/path/sp1\",\n                    \"user@host\"]\n    for ri in ssh_locators:\n        ok_(is_ssh(ri), \"not considered ssh (string): %s\" % ri)\n        ok_(is_ssh(RI(ri)), \"not considered ssh (RI): %s\" % ri)\n\n    non_ssh_locators = [\"file://path/to\",\n                        \"/abs/path\",\n                        \"../rel/path\",\n                        \"http://example.com\",\n                        \"git://host/user/proj\",\n                        \"s3://bucket/save/?key=891\"]\n    for ri in non_ssh_locators:\n        ok_(not is_ssh(ri), \"considered ssh (string): %s\" % ri)\n        ok_(not is_ssh(RI(ri)), \"considered ssh (RI): %s\" % ri)\n\n\ndef test_iso8601_to_epoch():\n    epoch = 1467901515\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15+00:00'), epoch)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15+11:00'),\n        epoch - 11 * 60 * 60)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15Z'), epoch)\n    eq_(iso8601_to_epoch('2016-07-07T14:25:15'), epoch)\n\n    eq_(iso8601_to_epoch('2016-07-07T14:25:14'), epoch-1)\n\n\ndef test_mapping_identity():\n    from datalad.tests.utils_pytest import OBSCURE_FILENAME\n\n    absolute_obscure_path = str(Path('/').absolute() / OBSCURE_FILENAME)\n    temp_dir = tempfile.gettempdir()\n    print(f\"temp_dir: {temp_dir}\")\n    for name in (temp_dir, opj(temp_dir, \"x.txt\"), absolute_obscure_path):\n        # On some platforms, e.g. MacOS, `temp_dir` might contain trailing\n        # slashes. Since the conversion and its inverse normalize paths, we\n        # compare the result to the normalized path\n        normalized_name = str(Path(name))\n        assert url_path2local_path(local_path2url_path(name)) == normalized_name\n\n    prefix = \"/C:\" if on_windows else \"\"\n    for name in map(quote_path, (prefix + \"/window\", prefix + \"/d\", prefix + \"/\" + OBSCURE_FILENAME)):\n        assert local_path2url_path(url_path2local_path(name)) == name\n\n\ndef test_auto_resolve_path():\n    relative_path = str(Path(\"a/b\"))\n    with pytest.raises(ValueError):\n        local_path2url_path(relative_path)\n    local_path2url_path(\"\", allow_relative_path=True)\n\n\n@skip_if(not on_windows)\ndef test_hostname_detection():\n    with pytest.raises(ValueError):\n        local_path2url_path(\"\\\\\\\\server\\\\share\\\\path\")\n\n\ndef test_url_path2local_path_excceptions():\n    with pytest.raises(ValueError):\n        url_path2local_path('')\n    with pytest.raises(ValueError):\n        url_path2local_path(None)\n    with pytest.raises(ValueError):\n        url_path2local_path('a/b')\n    with pytest.raises(ValueError):\n        url_path2local_path(PurePosixPath('a/b'))\n    with pytest.raises(ValueError):\n        url_path2local_path(PurePosixPath('//a/b'))\n\n\ndef test_quote_path(monkeypatch):\n    with monkeypatch.context() as ctx:\n        ctx.setattr(datalad.support.network, 'on_windows', True)\n        assert quote_path(\"/c:/win:xxx\") == \"/c:/win%3Axxx\"\n        assert quote_path(\"/C:/win:xxx\") == \"/C:/win%3Axxx\"\n\n        ctx.setattr(datalad.support.network, 'on_windows', False)\n        assert quote_path(\"/c:/win:xxx\") == \"/c%3A/win%3Axxx\"\n        assert quote_path(\"/C:/win:xxx\") == \"/C%3A/win%3Axxx\"\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.17.9+109.g4c614de1c\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 49ms\nInstalled 25 packages in 4.77s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 72 packages in 4.16s\nPrepared 1 package in 4.83s\nInstalled 59 packages in 549ms\n + annexremote==1.6.6\n + argcomplete==3.5.3\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==4.0.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.17.9+109.g4c614de1c (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4c614de1c1e352dddbd135961c0ae4595eb3053f)\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + mypy==0.991\n + mypy-extensions==1.0.0\n + patool==1.15.0\n + pillow==11.0.0\n + platformdirs==4.3.6\n + propcache==0.2.1\n + pycparser==2.22\n + pyperclip==1.9.0\n + pytest-fail-slow==0.6.0\n + python-gitlab==5.3.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + requests-toolbelt==1.0.0\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + soupsieve==2.6\n + tqdm==4.67.1\n + types-python-dateutil==2.9.0.20241206\n + types-requests==2.32.0.20241016\n + urllib3==2.3.0\n + vcrpy==4.3.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 104 packages in 292ms\nInstalled 32 packages in 871ms\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + execnet==2.1.1\n + filelock==3.16.1\n + gprof2dot==2024.6.6\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + pytest-xdist==3.6.1\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4c614de1c1e352dddbd135961c0ae4595eb3053f\nconfigfile: tox.ini\nplugins: fail-slow-0.6.0, xdist-3.6.1, cov-6.0.0\ncollected 37 items\n\nr2e_tests/test_1.py ................ss...........F....s..                [100%]\n\n=================================== FAILURES ===================================\n____________________ test_get_local_file_url_compatibility _____________________\n\npath = PosixPath('/tmp/datalad_temp_test_get_local_file_url_compatibilityzwjnr5cj')\n\n    @with_tempfile(mkdir=True)\n    def test_get_local_file_url_compatibility(path=None):\n        # smoke test for file:// URL compatibility with other datalad/git/annex\n        # pieces\n        path = Path(path)\n        ds1 = Dataset(path / 'ds1').create()\n        ds2 = Dataset(path / 'ds2').create()\n        testfile = path / 'testfile.txt'\n        testfile.write_text('some')\n    \n        # compat with annex addurl\n>       ds1.repo.add_url_to_file(\n            'test.txt',\n            get_local_file_url(str(testfile), compatibility='git-annex'))\n\nr2e_tests/test_1.py:546: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/support/gitrepo.py:238: in _wrap_normalize_path\n    return func(self, file_new, *args, **kwargs)\ndatalad/support/annexrepo.py:2145: in add_url_to_file\n    out_json = self._call_annex_records(\ndatalad/support/annexrepo.py:1076: in _call_annex_records\n    raise e\ndatalad/support/annexrepo.py:1040: in _call_annex_records\n    out = self._call_annex(\ndatalad/support/annexrepo.py:943: in _call_annex\n    return runner.run(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <datalad.runner.gitrunner.GitWitlessRunner object at 0x790d8f3b0dc0>\ncmd = ['git', '-c', 'diff.ignoreSubmodules=none', 'annex', 'addurl', '--file=test.txt', ...]\nprotocol = <class 'datalad.support.annexrepo.AnnexJsonProtocol'>, stdin = None\ncwd = None, env = None, timeout = None, exception_on_error = True, kwargs = {}\napplied_cwd = PosixPath('/tmp/datalad_temp_test_get_local_file_url_compatibilityzwjnr5cj/ds1')\napplied_env = {'ANTHROPIC_API_KEY': 'sk-ant-api03-kztdHBP1RruInQ2uzw8m4wJLbhPHBrb-qHX2mhAKmwoMFvM_dI2oWb5V6JtS45MScuW5CCjPusIrt9Zatx...3b4166bef66ef252935c/server/bin/helpers/browser.sh', 'COHERE_API_KEY': '66TDU3MmN5ELJT1NHWklGG2PJjK2zwVSQbVq42ol', ...}\nthreaded_runner = <datalad.runner.nonasyncrunner.ThreadedRunner object at 0x790d8fe18460>\nresults_or_iterator = {'code': 1, 'stderr': 'git-annex: addurl: 1 failed\\n', 'stdout': '', 'stdout_json': [{'command': 'addurl', 'error-mess...': 'test.txt', 'input': ['file:///tmp/datalad_temp_test_get_local_file_url_compatibilityzwjnr5cj/testfile.txt'], ...}]}\nresults = {'code': 1, 'stderr': 'git-annex: addurl: 1 failed\\n', 'stdout': '', 'stdout_json': [{'command': 'addurl', 'error-mess...': 'test.txt', 'input': ['file:///tmp/datalad_temp_test_get_local_file_url_compatibilityzwjnr5cj/testfile.txt'], ...}]}\n\n    def run(self,\n            cmd: list | str,\n            protocol: type[WitlessProtocol] | None = None,\n            stdin: bytes | IO | Queue | None = None,\n            cwd: PathLike | str | None = None,\n            env: dict | None = None,\n            timeout: float | None = None,\n            exception_on_error: bool = True,\n            **kwargs) -> dict | _ResultGenerator:\n        \"\"\"Execute a command and communicate with it.\n    \n        Parameters\n        ----------\n        cmd : list or str\n          Sequence of program arguments. Passing a single string causes\n          execution via the platform shell.\n        protocol : WitlessProtocol, optional\n          Protocol class handling interaction with the running process\n          (e.g. output capture). A number of pre-crafted classes are\n          provided (e.g `KillOutput`, `NoCapture`, `GitProgress`).\n          If the protocol has the GeneratorMixIn-mixin, the run-method\n          will return an iterator and can therefore be used in a for-clause.\n        stdin : file-like, bytes, Queue, or None\n          If stdin is a file-like, it will be directly used as stdin for the\n          subprocess. The caller is responsible for writing to it and closing it.\n          If stdin is a bytes, it will be fed to stdin of the subprocess.\n          If all data is written, stdin will be closed.\n          If stdin is a Queue, all elements (bytes) put into the Queue will\n          be passed to stdin until None is read from the queue. If None is read,\n          stdin of the subprocess is closed.\n        cwd : str or path-like, optional\n          If given, commands are executed with this path as PWD,\n          the PWD of the parent process is used otherwise. Overrides\n          any `cwd` given to the constructor.\n        env : dict, optional\n          Environment to be used for command execution. If given, it will\n          completely replace any environment provided to theconstructor. If\n          `cwd` is given, 'PWD' in the environment is set to its value.\n          This must be a complete environment definition, no values\n          from the current environment will be inherited. Overrides\n          any `env` given to the constructor.\n        timeout: float, optional\n          None or the seconds after which a timeout callback is\n          invoked, if no progress was made in communicating with\n          the sub-process, or if waiting for the subprocess exit\n          took more than the specified time. See the protocol and\n          `ThreadedRunner` descriptions for a more detailed discussion\n          on timeouts.\n        exception_on_error : bool, optional\n          This argument is only interpreted if the protocol is a subclass\n          of `GeneratorMixIn`. If it is `True` (default), a\n          `CommandErrorException` is raised by the generator if the\n          sub process exited with a return code not equal to zero. If the\n          parameter is `False`, no exception is raised. In both cases the\n          return code can be read from the attribute `return_code` of\n          the generator.\n        kwargs :\n          Passed to the Protocol class constructor.\n    \n        Returns\n        -------\n          dict | _ResultGenerator\n    \n            If the protocol is not a subclass of `GeneratorMixIn`, the\n            result of protocol._prepare_result will be returned.\n    \n            If the protocol is a subclass of `GeneratorMixIn`, a Generator, i.e.\n            a `_ResultGenerator`, will be returned. This allows to use this\n            method in constructs like:\n    \n                for protocol_output in runner.run():\n                    ...\n    \n            Where the iterator yields whatever protocol.pipe_data_received\n            sends into the generator.\n            If all output was yielded and the process has terminated, the\n            generator will raise StopIteration(return_code), where\n            return_code is the return code of the process. The return code\n            of the process will also be stored in the \"return_code\"-attribute\n            of the runner. So you could write:\n    \n               gen = runner.run()\n               for file_descriptor, data in gen:\n                   ...\n    \n               # get the return code of the process\n               result = gen.return_code\n    \n        Raises\n        ------\n        CommandError\n          On execution failure (non-zero exit code) this exception is\n          raised which provides the command (cmd), stdout, stderr,\n          exit code (status), and a message identifying the failed\n          command, as properties.\n        FileNotFoundError\n          When a given executable does not exist.\n        \"\"\"\n        if protocol is None:\n            # by default let all subprocess stream pass through\n            protocol = NoCapture\n    \n        applied_cwd = cwd or self.cwd\n        applied_env = self._get_adjusted_env(\n            env=env or self.env,\n            cwd=applied_cwd,\n        )\n    \n        lgr.debug(\n            'Run %r (protocol_class=%s) (cwd=%s)',\n            cmd,\n            protocol.__name__,\n            applied_cwd\n        )\n    \n        threaded_runner = ThreadedRunner(\n            cmd=cmd,\n            protocol_class=protocol,\n            stdin=stdin,\n            protocol_kwargs=kwargs,\n            timeout=timeout,\n            exception_on_error=exception_on_error,\n            cwd=applied_cwd,\n            env=applied_env\n        )\n    \n        results_or_iterator = threaded_runner.run()\n        if issubclass(protocol, GeneratorMixIn):\n            return results_or_iterator\n    \n        results = cast(dict, results_or_iterator)\n        # log before any exception is raised\n        lgr.debug(\"Finished %r with status %s\", cmd, results['code'])\n    \n        # make it such that we always blow if a protocol did not report\n        # a return code at all\n        if results.get('code', True) not in [0, None]:\n            # the runner has a better idea, doc string warns Protocol\n            # implementations not to return these\n            results.pop('cmd', None)\n            results.pop('cwd', None)\n>           raise CommandError(\n                # whatever the results were, we carry them forward\n                cmd=cmd,\n                cwd=applied_cwd,\n                **results,\n            )\nE           datalad.runner.exception.CommandError: CommandError: 'git -c diff.ignoreSubmodules=none annex addurl --file=test.txt file:///tmp/datalad_temp_test_get_local_file_url_compatibilityzwjnr5cj/testfile.txt --json --json-error-messages --json-progress -c annex.dotfiles=true' failed with exitcode 1 under /tmp/datalad_temp_test_get_local_file_url_compatibilityzwjnr5cj/ds1 [info keys: stdout_json]\nE           >  Configuration does not allow accessing file:///tmp/datalad_temp_test_get_local_file_url_compatibilityzwjnr5cj/testfile.txt [err: 'git-annex: addurl: 1 failed']\n\ndatalad/runner/runner.py:220: CommandError\n----------------------------- Captured stdout call -----------------------------\ncreate(ok): . (dataset)\ncreate(ok): . (dataset)\n----------------------------- Captured stderr call -----------------------------\n[INFO] scanning for unlocked files (this may take some time) \n[INFO] scanning for unlocked files (this may take some time) \n------------------------------ Captured log call -------------------------------\nINFO     datalad.annex:annexrepo.py:3902 scanning for unlocked files (this may take some time)\nINFO     datalad.ui.dialog:log.py:431 Clear progress bars\nINFO     datalad.ui.dialog:log.py:431 Refresh progress bars\nINFO     datalad.annex:annexrepo.py:3902 scanning for unlocked files (this may take some time)\nINFO     datalad.ui.dialog:log.py:431 Clear progress bars\nINFO     datalad.ui.dialog:log.py:431 Refresh progress bars\n==================================== PASSES ====================================\n________________________________ test_url_base _________________________________\n------------------------------ Captured log call -------------------------------\nWARNING  datalad.network:network.py:680 ParseResults contains params 'param', which will be ignored\n_____________________________ test_pathri_guessing _____________________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.network:network.py:680 ParseResults contains params 'param', which will be ignored\n___________________________ test_get_local_file_url ____________________________\n----------------------------- Captured stdout call -----------------------------\nD: test.txt\nD:  |;&%b5{}'\"<>ΔЙקم๗あ .datc \nD: /a\nD: /a/b/c\nD: /a~\nD: /a b/name\n____________________________ test_mapping_identity _____________________________\n----------------------------- Captured stdout call -----------------------------\ntemp_dir: /tmp\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_same_website\nPASSED r2e_tests/test_1.py::test_get_tld\nPASSED r2e_tests/test_1.py::test_dlurljoin\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[#]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[#tag]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[#tag/obscure]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[?param=1]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[?param=1&another=/]\nPASSED r2e_tests/test_1.py::test_rfc2822_to_epoch\nPASSED r2e_tests/test_1.py::test_get_response_disposition_filename\nPASSED r2e_tests/test_1.py::test_parse_url_opts\nPASSED r2e_tests/test_1.py::test_split_colon\nPASSED r2e_tests/test_1.py::test_url_eq\nPASSED r2e_tests/test_1.py::test_url_base\nPASSED r2e_tests/test_1.py::test_pathri_guessing\nPASSED r2e_tests/test_1.py::test_git_transport_ri\nPASSED r2e_tests/test_1.py::test_url_quote_path[SSHRI-clskwargs0-example.com:/ \"';a&b&cd `| ]\nPASSED r2e_tests/test_1.py::test_url_quote_path[URL-clskwargs1-http://example.com/%20%22%27%3Ba%26b%26cd%20%60%7C%20]\nPASSED r2e_tests/test_1.py::test_url_quote_path[PathRI-clskwargs2-/ \"';a&b&cd `| ]\nPASSED r2e_tests/test_1.py::test_url_compose_archive_one\nPASSED r2e_tests/test_1.py::test_url_fragments_and_query\nPASSED r2e_tests/test_1.py::test_url_dicts\nPASSED r2e_tests/test_1.py::test_get_url_path_on_fileurls\nPASSED r2e_tests/test_1.py::test_is_url\nPASSED r2e_tests/test_1.py::test_is_datalad_compat_ri\nPASSED r2e_tests/test_1.py::test_get_local_file_url\nPASSED r2e_tests/test_1.py::test_is_ssh\nPASSED r2e_tests/test_1.py::test_iso8601_to_epoch\nPASSED r2e_tests/test_1.py::test_mapping_identity\nPASSED r2e_tests/test_1.py::test_auto_resolve_path\nPASSED r2e_tests/test_1.py::test_url_path2local_path_excceptions\nPASSED r2e_tests/test_1.py::test_quote_path\nSKIPPED [2] datalad/tests/utils_pytest.py:336: condition was True\nSKIPPED [1] .venv/lib/python3.9/site-packages/_pytest/unittest.py:385: TODO: file://::1/some does complain about parsed version dropping ::1\nFAILED r2e_tests/test_1.py::test_get_local_file_url_compatibility - datalad.r...\n=================== 1 failed, 33 passed, 3 skipped in 5.27s ====================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_4c614de1c1e352dddbd135961c0ae4595eb3053f\nconfigfile: tox.ini\nplugins: fail-slow-0.6.0, xdist-3.6.1, cov-6.0.0\ncollected 37 items\n\nr2e_tests/test_1.py ................ss...........F...Fs..                [100%]\n\n=================================== FAILURES ===================================\n____________________ test_get_local_file_url_compatibility _____________________\n\npath = PosixPath('/tmp/datalad_temp_test_get_local_file_url_compatibilityyr8svye1')\n\n    @with_tempfile(mkdir=True)\n    def test_get_local_file_url_compatibility(path=None):\n        # smoke test for file:// URL compatibility with other datalad/git/annex\n        # pieces\n        path = Path(path)\n        ds1 = Dataset(path / 'ds1').create()\n        ds2 = Dataset(path / 'ds2').create()\n        testfile = path / 'testfile.txt'\n        testfile.write_text('some')\n    \n        # compat with annex addurl\n>       ds1.repo.add_url_to_file(\n            'test.txt',\n            get_local_file_url(str(testfile), compatibility='git-annex'))\n\nr2e_tests/test_1.py:546: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/support/gitrepo.py:238: in _wrap_normalize_path\n    return func(self, file_new, *args, **kwargs)\ndatalad/support/annexrepo.py:2145: in add_url_to_file\n    out_json = self._call_annex_records(\ndatalad/support/annexrepo.py:1076: in _call_annex_records\n    raise e\ndatalad/support/annexrepo.py:1040: in _call_annex_records\n    out = self._call_annex(\ndatalad/support/annexrepo.py:943: in _call_annex\n    return runner.run(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <datalad.runner.gitrunner.GitWitlessRunner object at 0x75f80e0e1f80>\ncmd = ['git', '-c', 'diff.ignoreSubmodules=none', 'annex', 'addurl', '--file=test.txt', ...]\nprotocol = <class 'datalad.support.annexrepo.AnnexJsonProtocol'>, stdin = None\ncwd = None, env = None, timeout = None, exception_on_error = True, kwargs = {}\napplied_cwd = PosixPath('/tmp/datalad_temp_test_get_local_file_url_compatibilityyr8svye1/ds1')\napplied_env = {'ANTHROPIC_API_KEY': 'sk-ant-api03-kztdHBP1RruInQ2uzw8m4wJLbhPHBrb-qHX2mhAKmwoMFvM_dI2oWb5V6JtS45MScuW5CCjPusIrt9Zatx...3b4166bef66ef252935c/server/bin/helpers/browser.sh', 'COHERE_API_KEY': '66TDU3MmN5ELJT1NHWklGG2PJjK2zwVSQbVq42ol', ...}\nthreaded_runner = <datalad.runner.nonasyncrunner.ThreadedRunner object at 0x75f80e0f3040>\nresults_or_iterator = {'code': 1, 'stderr': 'git-annex: addurl: 1 failed\\n', 'stdout': '', 'stdout_json': [{'command': 'addurl', 'error-mess...': 'test.txt', 'input': ['file:///tmp/datalad_temp_test_get_local_file_url_compatibilityyr8svye1/testfile.txt'], ...}]}\nresults = {'code': 1, 'stderr': 'git-annex: addurl: 1 failed\\n', 'stdout': '', 'stdout_json': [{'command': 'addurl', 'error-mess...': 'test.txt', 'input': ['file:///tmp/datalad_temp_test_get_local_file_url_compatibilityyr8svye1/testfile.txt'], ...}]}\n\n    def run(self,\n            cmd: list | str,\n            protocol: type[WitlessProtocol] | None = None,\n            stdin: bytes | IO | Queue | None = None,\n            cwd: PathLike | str | None = None,\n            env: dict | None = None,\n            timeout: float | None = None,\n            exception_on_error: bool = True,\n            **kwargs) -> dict | _ResultGenerator:\n        \"\"\"Execute a command and communicate with it.\n    \n        Parameters\n        ----------\n        cmd : list or str\n          Sequence of program arguments. Passing a single string causes\n          execution via the platform shell.\n        protocol : WitlessProtocol, optional\n          Protocol class handling interaction with the running process\n          (e.g. output capture). A number of pre-crafted classes are\n          provided (e.g `KillOutput`, `NoCapture`, `GitProgress`).\n          If the protocol has the GeneratorMixIn-mixin, the run-method\n          will return an iterator and can therefore be used in a for-clause.\n        stdin : file-like, bytes, Queue, or None\n          If stdin is a file-like, it will be directly used as stdin for the\n          subprocess. The caller is responsible for writing to it and closing it.\n          If stdin is a bytes, it will be fed to stdin of the subprocess.\n          If all data is written, stdin will be closed.\n          If stdin is a Queue, all elements (bytes) put into the Queue will\n          be passed to stdin until None is read from the queue. If None is read,\n          stdin of the subprocess is closed.\n        cwd : str or path-like, optional\n          If given, commands are executed with this path as PWD,\n          the PWD of the parent process is used otherwise. Overrides\n          any `cwd` given to the constructor.\n        env : dict, optional\n          Environment to be used for command execution. If given, it will\n          completely replace any environment provided to theconstructor. If\n          `cwd` is given, 'PWD' in the environment is set to its value.\n          This must be a complete environment definition, no values\n          from the current environment will be inherited. Overrides\n          any `env` given to the constructor.\n        timeout: float, optional\n          None or the seconds after which a timeout callback is\n          invoked, if no progress was made in communicating with\n          the sub-process, or if waiting for the subprocess exit\n          took more than the specified time. See the protocol and\n          `ThreadedRunner` descriptions for a more detailed discussion\n          on timeouts.\n        exception_on_error : bool, optional\n          This argument is only interpreted if the protocol is a subclass\n          of `GeneratorMixIn`. If it is `True` (default), a\n          `CommandErrorException` is raised by the generator if the\n          sub process exited with a return code not equal to zero. If the\n          parameter is `False`, no exception is raised. In both cases the\n          return code can be read from the attribute `return_code` of\n          the generator.\n        kwargs :\n          Passed to the Protocol class constructor.\n    \n        Returns\n        -------\n          dict | _ResultGenerator\n    \n            If the protocol is not a subclass of `GeneratorMixIn`, the\n            result of protocol._prepare_result will be returned.\n    \n            If the protocol is a subclass of `GeneratorMixIn`, a Generator, i.e.\n            a `_ResultGenerator`, will be returned. This allows to use this\n            method in constructs like:\n    \n                for protocol_output in runner.run():\n                    ...\n    \n            Where the iterator yields whatever protocol.pipe_data_received\n            sends into the generator.\n            If all output was yielded and the process has terminated, the\n            generator will raise StopIteration(return_code), where\n            return_code is the return code of the process. The return code\n            of the process will also be stored in the \"return_code\"-attribute\n            of the runner. So you could write:\n    \n               gen = runner.run()\n               for file_descriptor, data in gen:\n                   ...\n    \n               # get the return code of the process\n               result = gen.return_code\n    \n        Raises\n        ------\n        CommandError\n          On execution failure (non-zero exit code) this exception is\n          raised which provides the command (cmd), stdout, stderr,\n          exit code (status), and a message identifying the failed\n          command, as properties.\n        FileNotFoundError\n          When a given executable does not exist.\n        \"\"\"\n        if protocol is None:\n            # by default let all subprocess stream pass through\n            protocol = NoCapture\n    \n        applied_cwd = cwd or self.cwd\n        applied_env = self._get_adjusted_env(\n            env=env or self.env,\n            cwd=applied_cwd,\n        )\n    \n        lgr.debug(\n            'Run %r (protocol_class=%s) (cwd=%s)',\n            cmd,\n            protocol.__name__,\n            applied_cwd\n        )\n    \n        threaded_runner = ThreadedRunner(\n            cmd=cmd,\n            protocol_class=protocol,\n            stdin=stdin,\n            protocol_kwargs=kwargs,\n            timeout=timeout,\n            exception_on_error=exception_on_error,\n            cwd=applied_cwd,\n            env=applied_env\n        )\n    \n        results_or_iterator = threaded_runner.run()\n        if issubclass(protocol, GeneratorMixIn):\n            return results_or_iterator\n    \n        results = cast(dict, results_or_iterator)\n        # log before any exception is raised\n        lgr.debug(\"Finished %r with status %s\", cmd, results['code'])\n    \n        # make it such that we always blow if a protocol did not report\n        # a return code at all\n        if results.get('code', True) not in [0, None]:\n            # the runner has a better idea, doc string warns Protocol\n            # implementations not to return these\n            results.pop('cmd', None)\n            results.pop('cwd', None)\n>           raise CommandError(\n                # whatever the results were, we carry them forward\n                cmd=cmd,\n                cwd=applied_cwd,\n                **results,\n            )\nE           datalad.runner.exception.CommandError: CommandError: 'git -c diff.ignoreSubmodules=none annex addurl --file=test.txt file:///tmp/datalad_temp_test_get_local_file_url_compatibilityyr8svye1/testfile.txt --json --json-error-messages --json-progress -c annex.dotfiles=true' failed with exitcode 1 under /tmp/datalad_temp_test_get_local_file_url_compatibilityyr8svye1/ds1 [info keys: stdout_json]\nE           >  Configuration does not allow accessing file:///tmp/datalad_temp_test_get_local_file_url_compatibilityyr8svye1/testfile.txt [err: 'git-annex: addurl: 1 failed']\n\ndatalad/runner/runner.py:220: CommandError\n----------------------------- Captured stdout call -----------------------------\ncreate(ok): . (dataset)\ncreate(ok): . (dataset)\n----------------------------- Captured stderr call -----------------------------\n[INFO] scanning for unlocked files (this may take some time) \n[INFO] scanning for unlocked files (this may take some time) \n------------------------------ Captured log call -------------------------------\nINFO     datalad.annex:annexrepo.py:3902 scanning for unlocked files (this may take some time)\nINFO     datalad.ui.dialog:log.py:431 Clear progress bars\nINFO     datalad.ui.dialog:log.py:431 Refresh progress bars\nINFO     datalad.annex:annexrepo.py:3902 scanning for unlocked files (this may take some time)\nINFO     datalad.ui.dialog:log.py:431 Clear progress bars\nINFO     datalad.ui.dialog:log.py:431 Refresh progress bars\n____________________________ test_auto_resolve_path ____________________________\n\n    def test_auto_resolve_path():\n        relative_path = str(Path(\"a/b\"))\n        with pytest.raises(ValueError):\n            local_path2url_path(relative_path)\n>       local_path2url_path(\"\", allow_relative_path=True)\nE       TypeError: local_path2url_path() got an unexpected keyword argument 'allow_relative_path'\n\nr2e_tests/test_1.py:614: TypeError\n==================================== PASSES ====================================\n________________________________ test_url_base _________________________________\n------------------------------ Captured log call -------------------------------\nWARNING  datalad.network:network.py:680 ParseResults contains params 'param', which will be ignored\n_____________________________ test_pathri_guessing _____________________________\n------------------------------ Captured log call -------------------------------\nDEBUG    datalad.network:network.py:680 ParseResults contains params 'param', which will be ignored\n___________________________ test_get_local_file_url ____________________________\n----------------------------- Captured stdout call -----------------------------\nD: test.txt\nD:  |;&%b5{}'\"<>ΔЙקم๗あ .datc \nD: /a\nD: /a/b/c\nD: /a~\nD: /a b/name\n____________________________ test_mapping_identity _____________________________\n----------------------------- Captured stdout call -----------------------------\ntemp_dir: /tmp\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_same_website\nPASSED r2e_tests/test_1.py::test_get_tld\nPASSED r2e_tests/test_1.py::test_dlurljoin\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[#]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[#tag]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[#tag/obscure]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[?param=1]\nPASSED r2e_tests/test_1.py::test_get_url_straight_filename[?param=1&another=/]\nPASSED r2e_tests/test_1.py::test_rfc2822_to_epoch\nPASSED r2e_tests/test_1.py::test_get_response_disposition_filename\nPASSED r2e_tests/test_1.py::test_parse_url_opts\nPASSED r2e_tests/test_1.py::test_split_colon\nPASSED r2e_tests/test_1.py::test_url_eq\nPASSED r2e_tests/test_1.py::test_url_base\nPASSED r2e_tests/test_1.py::test_pathri_guessing\nPASSED r2e_tests/test_1.py::test_git_transport_ri\nPASSED r2e_tests/test_1.py::test_url_quote_path[SSHRI-clskwargs0-example.com:/ \"';a&b&cd `| ]\nPASSED r2e_tests/test_1.py::test_url_quote_path[URL-clskwargs1-http://example.com/%20%22%27%3Ba%26b%26cd%20%60%7C%20]\nPASSED r2e_tests/test_1.py::test_url_quote_path[PathRI-clskwargs2-/ \"';a&b&cd `| ]\nPASSED r2e_tests/test_1.py::test_url_compose_archive_one\nPASSED r2e_tests/test_1.py::test_url_fragments_and_query\nPASSED r2e_tests/test_1.py::test_url_dicts\nPASSED r2e_tests/test_1.py::test_get_url_path_on_fileurls\nPASSED r2e_tests/test_1.py::test_is_url\nPASSED r2e_tests/test_1.py::test_is_datalad_compat_ri\nPASSED r2e_tests/test_1.py::test_get_local_file_url\nPASSED r2e_tests/test_1.py::test_is_ssh\nPASSED r2e_tests/test_1.py::test_iso8601_to_epoch\nPASSED r2e_tests/test_1.py::test_mapping_identity\nPASSED r2e_tests/test_1.py::test_url_path2local_path_excceptions\nPASSED r2e_tests/test_1.py::test_quote_path\nSKIPPED [2] datalad/tests/utils_pytest.py:336: condition was True\nSKIPPED [1] .venv/lib/python3.9/site-packages/_pytest/unittest.py:385: TODO: file://::1/some does complain about parsed version dropping ::1\nFAILED r2e_tests/test_1.py::test_get_local_file_url_compatibility - datalad.r...\nFAILED r2e_tests/test_1.py::test_auto_resolve_path - TypeError: local_path2ur...\n=================== 2 failed, 32 passed, 3 skipped in 2.43s ====================\n",
    "old_commit_res_stderr": ""
}
datalad/support/network.py datalad/support/tests/test_network.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]

**Title:** `get_local_file_url` Generates Different URI Than `Path.as_uri()` for Absolute Paths

**Description:**
When converting an absolute file path to a URL using `get_local_file_url`, the resulting URI does not match the one produced by `Path.as_uri()`. This inconsistency triggers a warning and may lead to unexpected behavior when handling file URLs.

**Example Code:**
```python
from datalad.support.network import get_local_file_url
from pathlib import Path

absolute_path = "/absolute/path/to/file"
generated_url = get_local_file_url(absolute_path)
expected_url = Path(absolute_path).as_uri()

print(generated_url)  # Outputs: file:///absolute/path/to/file
print(expected_url)   # Outputs: file:///absolute/path/to/file
```

**Expected Behavior:**
Both `generated_url` and `expected_url` should be identical, resulting in the same URI without any warnings.

**Actual Behavior:**
`get_local_file_url` produces a URI that differs from `Path.as_uri()`, leading to a warning:
```
WARNING: get_local_file_url(/absolute/path/to/file, ...) returned file:///absolute/path/to/file instead of file:///absolute/path/to/file
```

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/network.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 et:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
from __future__ import annotations

import logging

lgr = logging.getLogger('datalad.network')

lgr.log(5, "Importing support.network")
import calendar
import email.utils
import os
import pickle
import re
import time
from collections import OrderedDict
from hashlib import md5
from ntpath import splitdrive as win_splitdrive
from os.path import dirname
from os.path import join as opj
from pathlib import PurePosixPath
from urllib.error import URLError
from urllib.parse import (
    ParseResult,
    parse_qsl,
)
from urllib.parse import quote as urlquote
from urllib.parse import unquote as urlunquote
from urllib.parse import (
    quote,
    urlencode,
    urljoin,
    urlparse,
    urlsplit,
    urlunparse,
)
from urllib.request import (
    Request,
    url2pathname,
)

import iso8601

from datalad import (
    cfg,
    consts,
)
from datalad.support.cache import lru_cache
from datalad.support.exceptions import CapturedException
from datalad.utils import (
    Path,
    PurePath,
    ensure_bytes,
    ensure_dir,
    ensure_unicode,
    map_items,
    on_windows,
)

# !!! Lazily import requests where needed -- needs 30ms or so
# import requests


def local_path_representation(path: str) -> str:
    """Return an OS-specific representation of a Posix-style path

    With a posix path in the form of "a/b" this function will return "a/b" on
    Unix-like operating systems and "a\\b" on Windows-style operating systems.
    """
    return str(Path(path))


def local_url_path_representation(url_path: str) -> str:
    """Return an OS-specific representation of the path component in a file:-URL

    With a path component like "/c:/Windows" (i.e. from a URL that reads
    "file:///c:/Windows"), this function will return "/c:/Windows" on a
    Unix-like operating systems and "C:\\Windows" on Windows-like operating
    systems.
    """
    return url2pathname(quote(url_path))


def local_path_from_url(url: str) -> str:
    """Parse the url and extract an OS-specific local path representation"""
    return local_url_path_representation(urlparse(url).path)


def is_windows_path(path):
    win_split = win_splitdrive(path)
    # Note, that ntpath.splitdrive also deals with UNC paths. In that case
    # the "drive" wouldn't be a windows drive letter followed by a colon
    if win_split[0] and win_split[1] and \
            win_split[0].endswith(":") and len(win_split[0]) == 2:
        # seems to be a windows path
        return True
    return False


def get_response_disposition_filename(s):
    """Given a string s as from HTTP Content-Disposition field in the response
    return possibly present filename if any
    """
    if not s:
        return None
    # If the response has Content-Disposition, try to get filename from it
    cd = map(
        lambda x: x.strip().split('=', 1) if '=' in x else [x.strip(), ''],
        s.split(';')
    )
    # unify the key to be lower case and make it into a dict
    cd = dict([[x[0].lower()] + x[1:] for x in cd])
    if 'filename' in cd:
        filename = cd['filename'].strip("\"'")
        return filename
    return None


def get_url_disposition_filename(url, headers=None):
    """Get filename as possibly provided by the server in Content-Disposition
    """
    if headers is None:
        request = Request(url)
        r = retry_urlopen(request)
        # things are different in requests
        if 'requests.' in str(r.__class__):
            headers = r.headers
        else:
            headers = r.info()
    else:
        r = None
    try:
        return get_response_disposition_filename(headers.get('Content-Disposition', ''))
    finally:
        if r:
            r.close()


def get_url_straight_filename(url, strip=None, allowdir=False):
    """Get file/dir name of the last path component of the URL

    Parameters
    ----------
    strip: list, optional
      If provided, listed names will not be considered and their
      parent directory will be selected
    allowdir: bool, optional
      If url points to a "directory" (ends with /), empty string
      would be returned unless allowdir is True, in which case the
      name of the directory would be returned
    """
    path = urlunquote(urlsplit(url).path)
    path_parts = path.split('/')

    if allowdir:
        # strip empty ones
        while len(path_parts) > 1 and not path_parts[-1]:
            path_parts = path_parts[:-1]

    if strip:
        while path_parts and path_parts[-1] in strip:
            path_parts = path_parts[:-1]

    if path_parts:
        return path_parts[-1]
    else:
        return None


def get_url_filename(url, headers=None, strip=None):
    """Get filename from the url, first consulting server about Content-Disposition
    """
    filename = get_url_disposition_filename(url, headers)
    if filename:
        return filename
    return get_url_straight_filename(url, strip=strip)


def get_url_response_stamp(url, response_info):
    size, mtime = None, None
    if 'Content-length' in response_info:
        size = int(response_info['Content-length'])
    if 'Last-modified' in response_info:
        mtime = calendar.timegm(email.utils.parsedate(
            response_info['Last-modified']))
    return dict(size=size, mtime=mtime, url=url)


def get_tld(url):
    """Return top level domain from a url

    Parameters
    ----------
    url : str
    """
    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld
    if not url.strip():
        raise ValueError("Empty URL has no TLD")
    rec = urlsplit(url)
    if not rec.netloc:
        if not rec.scheme:
            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'
            return url.split('/', 1)[0]
        else:
            raise ValueError("It seems that only the scheme was provided without the net location/TLD")
    return rec.netloc


from email.utils import (
    mktime_tz,
    parsedate_tz,
)


def rfc2822_to_epoch(datestr):
    """Given rfc2822 date/time format, return seconds since epoch"""
    return mktime_tz(parsedate_tz(datestr))


def iso8601_to_epoch(datestr):
    """Given ISO 8601 date/time format, return in seconds since epoch

    iso8601 is used to parse properly the time zone information, which
    can't be parsed with standard datetime strptime
    """
    return calendar.timegm(iso8601.parse_date(datestr).utctimetuple())


def __urlopen_requests(url):
    # XXX Workaround for now for ... broken code
    if isinstance(url, Request):
        url = url.get_full_url()
    from requests import Session
    return Session().get(url, stream=True)


def retry_urlopen(url, retries=3):
    for t in range(retries):
        try:
            return __urlopen_requests(url)
        except URLError as e:
            lgr.warning("Received exception while reading %s: %s" % (url, e))
            if t == retries - 1:
                # if we have reached allowed number of retries -- reraise
                raise


def is_url_quoted(url):
    """Return whether URL looks being already quoted
    """
    try:
        url_ = urlunquote(url)
        return url != url_
    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)
        # MIH: ValueError?
        return False


def same_website(url_rec, u_rec):
    """Decide whether a link leads to external site

    Parameters
    ----------
    url_rec: ParseResult
      record for original url
    u_rec: ParseResult
      record for new url
    """
    if isinstance(url_rec, str):
        url_rec = urlparse(url_rec)
    if isinstance(u_rec, str):
        u_rec = urlparse(u_rec)
    return (url_rec.netloc == u_rec.netloc)
    # todo: collect more of sample cases.
    # disabled below check while working on ratholeradio, since links
    # could go to the parent and that is ok.  Figure out when it was
    # desired not to go to the parent -- we might need explicit option
    # and u_rec.path.startswith(url_rec.path)):


def dlurljoin(u_path, url):
    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO
    if url_rec.scheme:
        # independent full url, so just return it
        return url
    if u_path.endswith('/'):  # should here be also a scheme use?
        if url.startswith('/'):  # jump to the root
            u_path_rec = urlparse(u_path)
            return urljoin(urlunparse(
                (u_path_rec.scheme, u_path_rec.netloc, '', '', '', '')), url)
        else:
            return os.path.join(u_path, url)
    # TODO: recall where all this dirname came from and bring into the test
    return urljoin(os.path.dirname(u_path) + '/', url)


# TODO should it be a node maybe?
class SimpleURLStamper(object):
    """Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header
    """
    def __init__(self, mode='full'):
        self.mode = mode

    def __call__(self, url):
        # Extracted from above madness
        # TODO: add mode alike to 'relaxed' where we would not
        # care about content-disposition filename
        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way
        request = Request(url)

        # No traffic compression since we do not know how to identify
        # exactly either it has to be decompressed
        # request.add_header('Accept-encoding', 'gzip,deflate')
        #
        # TODO: think about stamping etc -- we seems to be redoing
        # what git-annex does for us already... not really
        r = retry_urlopen(request)
        try:
            r_info = r.info()
            r_stamp = get_url_response_stamp(url, r_info)

            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)
        finally:
            r.close()


# TODO:  make it consistent/clear at what stage % encoding/decoding happens!
# now it is a mix!

#
# Useful functionality in requests.models
#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance
#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...
#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc
#

@lru_cache(maxsize=100)
def _guess_ri_cls(ri):
    """Factory function which would determine which type of a ri a provided string is"""
    TYPES = {
        'url': URL,
        'ssh':  SSHRI,
        'file': PathRI,
        'datalad': DataLadRI,
        'git-transport': GitTransportRI,
    }
    if isinstance(ri, PurePath):
        lgr.log(5, "Detected file ri")
        return TYPES['file']
    if is_windows_path(ri):
        # OMG we got something from windows
        lgr.log(5, "Detected file ri")
        return TYPES['file']

    # We assume that it is a URL and parse it. Depending on the result
    # we might decide that it was something else ;)
    fields = URL._pr_to_fields(urlparse(ri), guessing=True)
    lgr.log(5, "Parsed ri %s into fields %s", ri, fields)
    type_ = 'url'
    # Special treatments
    # file:///path should stay file:
    if fields['scheme'] and fields['scheme'] not in {'file'} \
            and not fields['hostname']:
        # transport::URL-or-path
        if fields['path'].startswith(':'):  # there was ::
            lgr.log(5, "Assuming git transport style ri and returning")
            type_ = 'git-transport'
        # dl+archive:... or just for ssh   hostname:path/p1
        elif '+' not in fields['scheme']:
            type_ = 'ssh'
            lgr.log(5, "Assuming ssh style ri, adjusted: %s", fields)

    if not fields['scheme'] and not fields['hostname']:
        parts = _split_colon(ri)
        # if no illegal for username@hostname characters in the first part and
        # we either had username@hostname or multiple :-separated parts
        if not set(parts[0]).intersection(set('/\\#')) and (
                fields['path'] and
                '@' in fields['path'] or
                len(parts) > 1
        ):
            # user@host:path/sp1
            # or host_name: (hence parts check)
            # TODO: we need a regex to catch those really, parts check is not suff
            type_ = 'ssh'
        elif ri.startswith('//'):
            # e.g. // or ///path
            type_ = 'datalad'
        else:
            type_ = 'file'

    if not fields['scheme'] and fields['hostname']:
        # e.g. //a/path
        type_ = 'datalad'

    cls = TYPES[type_]
    # just parse the ri according to regex matchint ssh "ri" specs
    lgr.log(5, "Detected %s ri", type_)
    return cls


class RI(object):
    """Resource Identifier - base class and a factory for URL, SSHRI, etc

    Intended to be a R/O object (i.e. no fields should be changed in-place).
    Subclasses define specific collections of fields they care about in _FIELDS
    class variable.
    The idea is that this class should help to break apart a URL, while being
    able to rebuild itself into a string representation for reuse

    `RI` could be used as factory, whenever type of the resource is unknown and
    must be guessed from the string representation.  One of the subclasses will be
    provided as output, e.g.

    >>> RI('http://example.com')
    URL(hostname='example.com', netloc='example.com', scheme='http')
    >>> RI('file://C:/Windows')
    URL(hostname='c', netloc='C:', path='/Windows', scheme='file')
    >>> RI('example.com:path')
    SSHRI(hostname='example.com', path='path')
    """

    # All of the subclasses will provide path
    _FIELDS = (
        'path',
    )

    __slots__ = _FIELDS + ('_fields', '_str')

    def __new__(cls, ri=None, **kwargs):
        """Used as a possible factory for known RI types

        Returns
        -------
        RI
           uninitialized RI object of appropriate class with _str
           set to string representation if was provided

        Raises
        ------
        ValueError
          Whenever the RI type cannot be determined.
        """
        if cls is RI and ri is not None:
            # RI class was used as a factory
            try:
                cls = _guess_ri_cls(ri)
            except Exception as e:
                # when anything goes wrong here, ensure a homogeneous
                # exception with a regular error
                raise ValueError(
                    f"Could not determine resource identifier type for {ri!r}"
                ) from e

        if cls is RI:
            raise ValueError(
                f"Could not determine resource identifier type for {ri!r}")

        ri_obj = super(RI, cls).__new__(cls)
        # Store internally original str
        ri_obj._str = str(ri) if isinstance(ri, PurePath) else ri
        return ri_obj

    def __init__(self, ri=None, **fields):
        """
        Parameters
        ----------
        ri: str, optional
          String version of a resource specific for this class.  If you would like
          a type of the resource be deduced, use RI(ri). Note that this value
          will be passed to str(), so you do not have to cast it yourself.
        **fields: dict, optional
          The values for the fields defined in _FIELDS class variable.
        """
        if ri and (bool(ri) == bool(fields)):
            raise ValueError(
                "Specify either ri or breakdown from the fields, not both. "
                "Got ri=%r, fields=%r" % (ri, fields))

        self._fields = self._get_blank_fields()
        if ri is not None:
            ri = str(ri)
            fields = self._str_to_fields(ri)
        self._set_from_fields(**fields)

        # If was initialized from a string representation
        if lgr.isEnabledFor(logging.DEBUG) and self._str is not None:
            # well -- some ris might not unparse identically back
            # strictly speaking, but let's assume they do
            ri_ = self.as_str()
            if ri != ri_:
                lgr.debug("Parsed version of %s %r differs from original %r",
                          self.__class__.__name__, ri_, ri)

    @classmethod
    def _get_blank_fields(cls, **fields):
        return OrderedDict(((f, fields.get(f, '')) for f in cls._FIELDS))

    @property
    def fields(self):
        """Returns shallow copy of fields to ease manipulations"""
        return self._fields.copy()

    def __repr__(self):
        # since auto_repr doesn't support "non-0" values atm
        return "%s(%s)" % (
            self.__class__.__name__,
            ", ".join(["%s=%r" % (k, v)
                       for k, v in sorted(self._fields.items())
                       if v]))

    # Lazily evaluated if _str was not set
    def __str__(self):
        if self._str is None:
            self._str = self.as_str()
        return self._str

    @classmethod
    def from_str(cls, ri_str):
        obj = cls(**cls._str_to_fields(ri_str))
        obj._str = ri_str
        return obj

    @property
    def localpath(self):
        # by default RIs point to remote locations
        raise ValueError("%s points to remote location" % self)

    # Apparently doesn't quite play nicely with multiple inheritance for MixIn'
    # of regexp based URLs
    #@abstractmethod
    #@classmethod
    #def _str_to_fields(cls, ri_str):
    #    raise NotImplementedError

    #
    # If any field is specified, URL is not considered 'False', i.e.
    # non-existing, although may be we could/shout omit having only
    # scheme or port specified since it doesn't point to any useful
    # location
    #

    def __bool__(self):
        fields = self._fields
        return any(fields.values())

    #
    # Helpers to deal with internal structures and conversions
    #

    def _set_from_fields(self, **fields):
        unknown_fields = set(fields).difference(self._FIELDS)
        if unknown_fields:
            raise ValueError("Do not know about %s. Known fields for %s are: %s"
                             % (unknown_fields, self.__class__, self._FIELDS))

        # encode dicts for query or fragment into
        for f in {'query', 'fragment'}:
            v = fields.get(f)
            if isinstance(v, dict):

                ev = urlencode(map_items(ensure_bytes, v))
                # / is reserved char within query
                if f == 'fragment' and '%2F' not in str(v):
                    # but seems to be ok'ish within the fragment which is
                    # the last element of URI and anyways used only by the
                    # client (i.e. by us here if used to compose the URL)
                    # so let's return / back for clarity if there were no
                    # awkward %2F to startswith
                    ev = ev.replace('%2F', '/')
                fields[f] = ev

        self._fields.update(fields)

    #
    # Quick comparators
    #

    def __eq__(self, other):
        if not isinstance(other, RI):
            other = RI(other)
        return isinstance(other, self.__class__) and dict(other._fields) == dict(self._fields)

    def __ne__(self, other):
        return not (self == other)

    def __getattribute__(self, item):
        if item.startswith('_') or item not in self._FIELDS:
            return super(RI, self).__getattribute__(item)
        else:
            return self._fields[item]

    def __setattr__(self, item, value):
        if item.startswith('_') or item not in self._FIELDS:
            super(RI, self).__setattr__(item, value)
        else:
            self._fields[item] = value
            self._str = None


class URL(RI):
    """Universal resource locator

    Although largely decorating urlparse.ParseResult, it
    - doesn't mandate providing all parts of the URL
    - doesn't require netloc but rather asks for separate username, password, and hostname
    """

    _FIELDS = RI._FIELDS + (
        'scheme',
        'netloc',
        'username',
        'password',
        'hostname', 'port',
        'query',
        'fragment',
    )

    # Only interpreted on Windows. If set to `True`, UNC-names encoded in
    # file-URLs, e.g. "file://server/share/path", would be considered local
    # and mapped onto "\\server\share\path". If set to `False`, UNC-names
    # encoded in file-URLs would not be considered local and cannot be resolved
    # to a local path.
    support_unc = False

    def as_str(self):
        """Render URL as a string"""
        return urlunparse(self.to_pr())

    @classmethod
    def _str_to_fields(cls, url_str):
        fields = URL._pr_to_fields(urlparse(url_str))
        fields['path'] = urlunquote(fields['path'])
        return fields

    def to_pr(self):
        """Convert URL to urlparse.ParseResults namedtuple"""
        return self._fields_to_pr(self._fields)

    @classmethod
    def _fields_to_pr(cls, fields):
        """Recompose back fields dict to ParseResult"""
        netloc = fields['username'] or ''
        if fields['password']:
            netloc += ':' + fields['password']
        if netloc:
            netloc += '@'
        netloc += fields['hostname']
        if fields['port']:
            if fields['hostname'].count(':') >= 2:
                # ipv6 -- need to enclose in []
                netloc = '[%s]:%s' % (netloc, fields['port'])
            else:
                netloc += ':%s' % fields['port']

        pr_fields = {
            f: fields[f]
            for f in cls._FIELDS
            if f not in ('hostname', 'password', 'username', 'port')
        }
        pr_fields['netloc'] = netloc
        pr_fields['params'] = ''
        # We need to quote the path
        pr_fields['path'] = urlquote(pr_fields['path'])
        # TODO: figure out what to do with query/fragment... one step at a time
        return ParseResult(**pr_fields)

    @classmethod
    def _pr_to_fields(cls, pr, guessing=False):
        """ParseResult is a tuple so immutable, which complicates adjusting it

        This function converts ParseResult into dict"""
        if pr.params:
            (lgr.debug if guessing else lgr.warning)(
                "ParseResults contains params %s, which will be ignored",
                repr(pr.params),)

        hostname_port = pr.netloc.split('@')[-1]
        is_ipv6 = hostname_port.count(':') >= 2
        # can't use just pr._asdict since we care to ask those properties
        # such as .port , .hostname etc
        # Forcing '' instead of None since those properties (.hostname), .password,
        # .username return None if not available and we decided to uniformize
        if is_ipv6:
            rem = re.match(r'\[(?P<hostname>.*)\]:(?P<port>\d+)', hostname_port)
            if rem:
                hostname, port = rem.groups()
                port = int(port)
            else:
                hostname, port = hostname_port, ''

            def _getattr(pr, f):
                """Helper for custom handling in case of ipv6 addresses which blows
                stock ParseResults logic"""
                if f == 'port':
                    # for now not supported at all, so
                    return port
                elif f == 'hostname':
                    return hostname
                else:
                    return getattr(pr, f)
        else:
            _getattr = getattr

        return {f: (_getattr(pr, f) or '') for f in cls._FIELDS}

    #
    # Access helpers
    #

    def _parse_qs(self, s, auto_delist=True):
        """Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values"""
        if not s:
            return {}
        out = map_items(ensure_unicode, OrderedDict(parse_qsl(s, 1)))
        if not auto_delist:
            return out
        for k in out:
            v = out[k]
            if isinstance(v, list) and len(v) == 1:
                v = v[0]
                out[k] = None if v == '' else v
        return out

    @property
    def query_dict(self):
        return self._parse_qs(self.query)

    @property
    def fragment_dict(self):
        return self._parse_qs(self.fragment)

    def _windows_local_path(self,
                            support_unc: bool = False
                            ) -> str:
        """Convert the URL to a local path on windows, supports UNC and git-annex"""

        # RFC1738 and RFC3986 both forbid unescaped backslash characters in
        # URLs, and therefore also in the path-component of file:-URLs. We
        # assume here that any backslash present in a file-URL is a relict of a
        # verbatim copy of a Windows-style path.
        unified_path = self.path.replace('\\', '/')
        local_path = url2pathname(unified_path)

        # We support UNC notation, and the "special" git-annex drive encoding
        # scheme, i.e. netloc is the drive letter plus a colon.
        # NB, this if clause will not evaluate to True, because our caller
        # filters out net locations with
        if self.netloc:
            if re.match('^[a-zA-Z]:$', self.netloc):
                # This is the git-annex case, i.e. drive spec in netloc
                return self.netloc + local_path
            if support_unc:
                return '\\\\' + self.netloc + local_path
            raise ValueError("Unsupported file: URL: {self}")
        return local_path

    @property
    def localpath(self):
        if self.scheme != 'file':
            raise ValueError(
                "Non 'file://' URL cannot be resolved to a local path")

        # If there is a hostname, we might want to convert to UNC, unless the
        # hostname has the form of a 'Windows drive letter' on windows
        hostname = self.hostname
        if not (hostname in (None, '', 'localhost', '::1')
                or hostname.startswith('127.')
                or re.match('^[a-zA-Z]:$', self.netloc)):
            if on_windows:
                return self._windows_local_path(support_unc=self.support_unc)
            raise ValueError("file:// URL does not point to 'localhost'")

        if on_windows:
            return self._windows_local_path(support_unc=True)

        return url2pathname(self.path)


class PathRI(RI):
    """RI pointing to a (local) file/directory"""
    def as_str(self):
        return self.path

    @classmethod
    def _str_to_fields(cls, url_str):
        # str() to be compatible with pathlib objects
        return dict(path=str(url_str))

    @property
    def localpath(self):
        return str(Path(self.path))

    @property
    def posixpath(self):
        if is_windows_path(self.path):
            win_split = win_splitdrive(self.path)
            return "/" + win_split[0][0] + win_split[1].replace('\\', '/')
        else:
            return self.path


class RegexBasedURLMixin(object):
    """Base class for URLs which we could simple parse using regular expressions"""

    _REGEX = None

    # not used ATM but possible ;)
    # @classmethod
    # def is_str_matches(cls, url_str):
    #     return bool(cls._REGEX.match(url_str))

    @classmethod
    def _str_to_fields(cls, url_str):
        re_match = cls._REGEX.match(url_str)
        if not re_match:
            # TODO: custom error?
            raise ValueError(
                "Cannot handle URL '%s': categorized as %r, but does not match syntax.%s"
                % (url_str,
                   cls,
                   " Did you intent to use '///'?" if url_str.startswith('//') else '')
            )
        fields = cls._get_blank_fields()
        fields.update({k: v for k, v in re_match.groupdict().items() if v})
        cls._normalize_fields(fields)
        return fields

    @classmethod
    def _normalize_fields(self, fields):
        """Helper to be ran if any of the fields need to be normalized after parsing"""
        pass


class SSHRI(RI, RegexBasedURLMixin):
    """RI pointing to a remote location reachable via SSH"""

    _FIELDS = RI._FIELDS + (
        'username',
        'hostname',
        'port',
    )

    _REGEX = re.compile(r'((?P<username>\S*)@)?(?P<hostname>[^#/\\:]+)(\:(?P<path>.*))?$')

    @classmethod
    def _normalize_fields(cls, fields):
        if fields['path'] and fields['path'].startswith('//'):
            # Let's normalize for now to avoid multiple leading slashes
            fields['path'] = '/' + fields['path'].lstrip('/')
        # escape path so we have direct representation of the path to work with
        fields['path'] = unescape_ssh_path(fields['path'])

    def as_str(self, escape=False):
        fields = self.fields  # copy so we could escape symbols
        url_fmt = '{hostname}'
        if fields['username']:
            url_fmt = "{username}@" + url_fmt
        if fields['path']:
            url_fmt += ':{path}'
        if escape:
            fields['path'] = escape_ssh_path(fields['path'])
        return url_fmt.format(**fields)

    # TODO:
    # we can "support" localhost:path as localpaths


class DataLadRI(RI, RegexBasedURLMixin):
    """RI pointing to datasets within default DataLad super-dataset"""

    _FIELDS = RI._FIELDS + (
        'remote',
    )

    # For now or forever we don't deal with any fragments or other special stuff
    _REGEX = re.compile(r'//(?P<remote>[^\s/]*)/(?P<path>.*)$')

    # do they need to be normalized??? losing track ...

    def as_str(self):
        return "//{remote}/{path}".format(**self._fields)

    def as_git_url(self):
        """Dereference /// into original URLs which could be used by git for cloning

        Returns
        -------
        str
          URL string to reference the DataLadRI from its /// form
        """
        if self.remote:
            raise NotImplementedError("not supported ATM to reference additional remotes")
        return "{}{}".format(consts.DATASETS_TOPURL, urlquote(self.path))


class GitTransportRI(RI, RegexBasedURLMixin):
    """RI for some other RI with git transport prefix"""

    # TODO: check how crticial to "inherit" RI._FIELDS asking to provide path
    _FIELDS = RI._FIELDS + (
        'transport',
        'RI',
    )

    # Due to poor design, `ri` argument already present in various
    # places intermixed with **kwargs treatment. So we will use RI
    # here instead of ri.
    _REGEX = re.compile(r'(?P<transport>[A-Za-z0-9][A-Za-z0-9+.-]*)::(?P<RI>.*)$')

    def as_str(self):
        return '{self.transport}::{self.RI}'.format(self=self)


def _split_colon(s, maxsplit=1):
    """Split on unescaped colon"""
    return re.compile(r'(?<!\\):').split(s, maxsplit=maxsplit)

# \ should be first to deal with
_SSH_ESCAPED_CHARACTERS = '\\#&;`|*?~<>^()[]{}$\'" '


# TODO: RF using re.sub
def escape_ssh_path(path):
    """Escape all special characters present in the path"""
    for c in _SSH_ESCAPED_CHARACTERS:
        if c in path:
            path = path.replace(c, '\\' + c)
    return path


def unescape_ssh_path(path):
    """Un-escape all special characters present in the path"""
    for c in _SSH_ESCAPED_CHARACTERS[::-1]:
        if c in path:
            path = path.replace('\\' + c, c)
    return path


def parse_url_opts(url):
    """Given a string with url-style query, split into content before # and options as dict"""
    url = URL(url)
    # we need to filter out query and fragment to get the base url
    fields = url.fields
    fields.pop('query')
    fields.pop('fragment')
    opts = url.query_dict
    return str(URL(**fields)), opts


# TODO: should we just define URL.good_for_git or smth like that? ;)
# although git also understands regular paths
def is_url(ri):
    """Returns whether argument is a resource identifier what datalad should treat as a URL

    This includes ssh "urls" which git understands.

    Parameters
    ----------
    ri : str or RI
      The resource identifier (as a string or RI) to "analyze"
    """
    if not isinstance(ri, RI):
        try:
            ri = RI(ri)
        except:  # MIH: MemoryError?
            return False
    return isinstance(ri, (URL, SSHRI))


# TODO: RF to remove duplication
def is_datalad_compat_ri(ri):
    """Returns whether argument is a resource identifier what datalad should treat as a URL

    including its own DataLadRI
    """
    if not isinstance(ri, RI):
        try:
            ri = RI(ri)
        except:  # MIH: MemoryError?
            return False
    return isinstance(ri, (URL, SSHRI, DataLadRI))


# TODO: better name? additionally may be move to SSHRI.is_valid() or sth.
def is_ssh(ri):
    """helper to determine, whether `ri` requires an SSH connection

    Parameters
    ----------
    ri: str or RI

    Returns
    -------
    bool
    """

    # not exactly fitting the doc, but we actually can deal not necessarily with
    # string or RI only, but with everything RI itself can deal with:
    _ri = RI(ri) if not isinstance(ri, RI) else ri

    return isinstance(_ri, SSHRI) \
        or (isinstance(_ri, URL) and _ri.scheme == 'ssh')


def get_local_file_url(fname: str,
                       compatibility: str = 'git-annex',
                       allow_relative_path: bool = True
                       ) -> str:
    """Return OS specific URL pointing to a local file

    Parameters
    ----------
    fname : string
        Filename.  If not absolute, abspath is used
    compatibility : str, optional
        This parameter is only interpreted on Windows systems. If set to
        anything else than 'git', the anchor, e.g. `C:` of `fname` will be put
        into the `file-auth` part, i.e. network location, defined in RFC 8089.
        This option is mainly used to support git-annex specific encoding of
        Windows paths.
    allow_relative_path: bool, optional
        Allow `fname` to be a relative path. The path will be converted to an
        absolute path, by using the current directory as path prefix.
    """
    path_obj = Path(fname)
    if not path_obj.is_absolute():
        if allow_relative_path:
            fname = str(Path(os.getcwd()) / path_obj)
        else:
            raise ValueError('cannot create file-URL with relative path')

    url_path = local_path2url_path(fname, auto_resolve=False)
    if on_windows and compatibility != "git":
        # Work around the way in which git-annex interprets file URLs on
        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`
        # into the network location component of the resulting URL.
        return "file:/" + url_path

    result = "file://" + url_path
    if result != Path(fname).as_uri():
        lgr.warning(
            f"get_local_file_url({fname}, {compatibility}, "
            f"{allow_relative_path}) returned {result} instead of "
            f"{Path(fname).as_uri()}")
    return result


def get_url_cache_filename(url, name=None):
    """Return a filename where to cache online doc from a url"""
    if not name:
        name = "misc"
    cache_dir = opj(cfg.obtain('datalad.locations.cache'), name)
    doc_fname = opj(
        cache_dir,
        '{}-{}.p{}'.format(
            urlsplit(url).netloc,
            md5(url.encode('utf-8')).hexdigest(),
            pickle.HIGHEST_PROTOCOL)
    )
    return doc_fname


def get_cached_url_content(url, name=None, fetcher=None, maxage=None):
    """Loader of a document from a url, which caches loaded instance on disk

    Doesn't do anything smart about http headers etc which could provide
    information for cache/proxy servers for how long to retain etc

    TODO: theoretically it is not network specific at all -- and just a memoize
    pattern, but may be some time we would make it treat headers etc correctly.
    And ATM would support any URL we support via providers/downloaders

    Parameters
    ----------
    fetcher: callable, optional
       Function to call with url if needed to be refetched
    maxage: float, optional
       Age in days to retain valid for.  <0 - would retain forever.  If None -
       would consult the config, 0 - would force to reload
    """
    doc_fname = get_url_cache_filename(url, name)
    if maxage is None:
        maxage = float(cfg.get('datalad.locations.cache-maxage'))

    doc = None
    if os.path.exists(doc_fname) and maxage != 0:

        fage = (time.time() - os.stat(doc_fname).st_mtime)/(24. * 3600)
        if maxage < 0 or fage < maxage:
            try:
                lgr.debug("use cached request result to '%s' from %s", url, doc_fname)
                doc = pickle.load(open(doc_fname, 'rb'))
            except Exception as e:  # it is OK to ignore any error and fall back on the true source
                lgr.warning(
                    "cannot load cache from '%s', fall back to download: %s",
                    doc_fname, CapturedException(e))

    if doc is None:
        if fetcher is None:
            from datalad.downloaders.providers import Providers
            providers = Providers.from_config_files()
            fetcher = providers.fetch

        doc = fetcher(url)
        ensure_dir(dirname(doc_fname))
        # use pickle to store the entire request result dict
        pickle.dump(doc, open(doc_fname, 'wb'))
        lgr.debug("stored result of request to '%s' in %s", url, doc_fname)
    return doc


def download_url(url, dest=None, overwrite=False):
    """Download a file from a URL

    Supports and honors any DataLad "downloader/provider" configuration.

    Parameters
    ----------
    url: str
      Source URL to download from.
    dest: Path-like or None
      Destination file name (file must not exist), or name of a target
      directory (must exists, and filename must be derivable from `url`).
      If None, the downloaded content will be returned as a string.
    overwrite: bool
      Force overwriting an existing destination file.

    Returns
    -------
    str
      Path of the downloaded file, or URL content if `dest` is None.

    Raises
    ------
    DownloadError
      If `dest` already exists and is a file, or if `dest` is a directory
      and no filename could be determined from `url`, or if no file was
      found at the given `url`.
    """
    from datalad.downloaders.providers import Providers
    providers = Providers.from_config_files()
    if dest:
        return providers.download(url, path=str(dest), overwrite=overwrite)
    else:
        return providers.fetch(url)


def local_path2url_path(local_path: str,
                        auto_resolve: bool = False
                        ) -> str:
    """Convert a local path into a URL path component"""
    if not local_path:
        if not auto_resolve:
            raise ValueError("cannot convert empty local path to URL path")
        local_path = os.getcwd()

    url = urlparse(Path(local_path).as_uri())
    if url.netloc:
        raise ValueError(
            f"cannot convert remote path to an URL path: {local_path}")
    return url.path


def url_path2local_path(url_path: str | PurePosixPath,
                        make_absolute: bool = True,
                        ) -> str | Path:

    if isinstance(url_path, PurePosixPath):
        return_path = True
        url_path = str(url_path)
    else:
        return_path = False

    if not url_path or not url_path.startswith("/"):
        # We expect a 'path-absolute' as defined in RFC 3986, therefore the
        # path must begin with a slash.
        raise ValueError(
            f"url path does not start with '/': {url_path}, and is therefore "
            f"not an absolute-path as defined in RFC 8089")

    if url_path.startswith("//"):
        # We expect a 'path-absolute' as defined in RFC 3986, therefore the
        # first segment must not be empty, i.e. the path must not start with
        # two or more slashes.
        raise ValueError(
            f"url path has empty first segment: {url_path}, and is therefore "
            f"not an absolute-path as defined in RFC 8089")

    return (
        Path(url2pathname(url_path))
        if return_path
        else url2pathname(url_path)
    )


def quote_path(path: str, safe: str = "/") -> str:
    """quote the path component of a URL, takes OS specifics into account"""
    if on_windows:
        if re.match("^/[a-zA-Z]:/", path):
            return path[:3] + quote(path[3:], safe=safe)
    return quote(path, safe=safe)


lgr.log(5, "Done importing support.network")

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]

**Title:** `get_local_file_url` Generates Different URI Than `Path.as_uri()` for Absolute Paths

**Description:**
When converting an absolute file path to a URL using `get_local_file_url`, the resulting URI does not match the one produced by `Path.as_uri()`. This inconsistency triggers a warning and may lead to unexpected behavior when handling file URLs.

**Example Code:**
```python
from datalad.support.network import get_local_file_url
from pathlib import Path

absolute_path = "/absolute/path/to/file"
generated_url = get_local_file_url(absolute_path)
expected_url = Path(absolute_path).as_uri()

print(generated_url)  # Outputs: file:///absolute/path/to/file
print(expected_url)   # Outputs: file:///absolute/path/to/file
```

**Expected Behavior:**
Both `generated_url` and `expected_url` should be identical, resulting in the same URI without any warnings.

**Actual Behavior:**
`get_local_file_url` produces a URI that differs from `Path.as_uri()`, leading to a warning:
```
WARNING: get_local_file_url(/absolute/path/to/file, ...) returned file:///absolute/path/to/file instead of file:///absolute/path/to/file
```

[/ISSUE]
{
    "test_same_website": "PASSED",
    "test_get_tld": "PASSED",
    "test_dlurljoin": "PASSED",
    "test_get_url_straight_filename[]": "PASSED",
    "test_get_url_straight_filename[#]": "PASSED",
    "test_get_url_straight_filename[#tag]": "PASSED",
    "test_get_url_straight_filename[#tag/obscure]": "PASSED",
    "test_get_url_straight_filename[?param=1]": "PASSED",
    "test_get_url_straight_filename[?param=1&another=/]": "PASSED",
    "test_rfc2822_to_epoch": "PASSED",
    "test_get_response_disposition_filename": "PASSED",
    "test_parse_url_opts": "PASSED",
    "test_split_colon": "PASSED",
    "test_url_eq": "PASSED",
    "test_url_base": "PASSED",
    "test_pathri_guessing": "PASSED",
    "test_git_transport_ri": "PASSED",
    "test_url_quote_path[SSHRI-clskwargs0-example.com:/ \"';a&b&cd `| ]": "PASSED",
    "test_url_quote_path[URL-clskwargs1-http://example.com/%20%22%27%3Ba%26b%26cd%20%60%7C%20]": "PASSED",
    "test_url_quote_path[PathRI-clskwargs2-/ \"';a&b&cd `| ]": "PASSED",
    "test_url_compose_archive_one": "PASSED",
    "test_url_fragments_and_query": "PASSED",
    "test_url_dicts": "PASSED",
    "test_get_url_path_on_fileurls": "PASSED",
    "test_is_url": "PASSED",
    "test_is_datalad_compat_ri": "PASSED",
    "test_get_local_file_url": "PASSED",
    "test_is_ssh": "PASSED",
    "test_iso8601_to_epoch": "PASSED",
    "test_mapping_identity": "PASSED",
    "test_auto_resolve_path": "PASSED",
    "test_url_path2local_path_excceptions": "PASSED",
    "test_quote_path": "PASSED",
    "test_get_local_file_url_compatibility": "FAILED"
}
diff --git a/datalad/support/network.py b/datalad/support/network.py
index 30270a20e..856eac792 100644
--- a/datalad/support/network.py
+++ b/datalad/support/network.py
@@ -1029,14 +1029,7 @@ def get_local_file_url(fname: str,
         Allow `fname` to be a relative path. The path will be converted to an
         absolute path, by using the current directory as path prefix.
     """
-    path_obj = Path(fname)
-    if not path_obj.is_absolute():
-        if allow_relative_path:
-            fname = str(Path(os.getcwd()) / path_obj)
-        else:
-            raise ValueError('cannot create file-URL with relative path')
-
-    url_path = local_path2url_path(fname, auto_resolve=False)
+    url_path = local_path2url_path(fname, allow_relative_path=allow_relative_path)
     if on_windows and compatibility != "git":
         # Work around the way in which git-annex interprets file URLs on
         # Windows. This code path will put the path anchor, e.g. `C:` of `fname`
@@ -1044,11 +1037,6 @@ def get_local_file_url(fname: str,
         return "file:/" + url_path
 
     result = "file://" + url_path
-    if result != Path(fname).as_uri():
-        lgr.warning(
-            f"get_local_file_url({fname}, {compatibility}, "
-            f"{allow_relative_path}) returned {result} instead of "
-            f"{Path(fname).as_uri()}")
     return result
 
 
@@ -1153,13 +1141,12 @@ def download_url(url, dest=None, overwrite=False):
 
 
 def local_path2url_path(local_path: str,
-                        auto_resolve: bool = False
+                        allow_relative_path: bool = False
                         ) -> str:
-    """Convert a local path into a URL path component"""
-    if not local_path:
-        if not auto_resolve:
-            raise ValueError("cannot convert empty local path to URL path")
-        local_path = os.getcwd()
+    """Convert a local path into an URL path component"""
+    local_path = Path(local_path)
+    if not local_path.is_absolute() and allow_relative_path:
+        local_path = local_path.absolute()
 
     url = urlparse(Path(local_path).as_uri())
     if url.netloc:
@@ -1168,10 +1155,7 @@ def local_path2url_path(local_path: str,
     return url.path
 
 
-def url_path2local_path(url_path: str | PurePosixPath,
-                        make_absolute: bool = True,
-                        ) -> str | Path:
-
+def url_path2local_path(url_path: str | PurePosixPath) -> str | Path:
     if isinstance(url_path, PurePosixPath):
         return_path = True
         url_path = str(url_path)

{"datalad/support/network.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 et:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\nfrom __future__ import annotations\n\nimport logging\n\nlgr = logging.getLogger('datalad.network')\n\nlgr.log(5, \"Importing support.network\")\nimport calendar\nimport email.utils\nimport os\nimport pickle\nimport re\nimport time\nfrom collections import OrderedDict\nfrom hashlib import md5\nfrom ntpath import splitdrive as win_splitdrive\nfrom os.path import dirname\nfrom os.path import join as opj\nfrom pathlib import PurePosixPath\nfrom urllib.error import URLError\nfrom urllib.parse import (\n    ParseResult,\n    parse_qsl,\n)\nfrom urllib.parse import quote as urlquote\nfrom urllib.parse import unquote as urlunquote\nfrom urllib.parse import (\n    quote,\n    urlencode,\n    urljoin,\n    urlparse,\n    urlsplit,\n    urlunparse,\n)\nfrom urllib.request import (\n    Request,\n    url2pathname,\n)\n\nimport iso8601\n\nfrom datalad import (\n    cfg,\n    consts,\n)\nfrom datalad.support.cache import lru_cache\nfrom datalad.support.exceptions import CapturedException\nfrom datalad.utils import (\n    Path,\n    PurePath,\n    ensure_bytes,\n    ensure_dir,\n    ensure_unicode,\n    map_items,\n    on_windows,\n)\n\n# !!! Lazily import requests where needed -- needs 30ms or so\n# import requests\n\n\ndef local_path_representation(path: str) -> str:\n    \"\"\"Return an OS-specific representation of a Posix-style path\n\n    With a posix path in the form of \"a/b\" this function will return \"a/b\" on\n    Unix-like operating systems and \"a\\\\b\" on Windows-style operating systems.\n    \"\"\"\n    return str(Path(path))\n\n\ndef local_url_path_representation(url_path: str) -> str:\n    \"\"\"Return an OS-specific representation of the path component in a file:-URL\n\n    With a path component like \"/c:/Windows\" (i.e. from a URL that reads\n    \"file:///c:/Windows\"), this function will return \"/c:/Windows\" on a\n    Unix-like operating systems and \"C:\\\\Windows\" on Windows-like operating\n    systems.\n    \"\"\"\n    return url2pathname(quote(url_path))\n\n\ndef local_path_from_url(url: str) -> str:\n    \"\"\"Parse the url and extract an OS-specific local path representation\"\"\"\n    return local_url_path_representation(urlparse(url).path)\n\n\ndef is_windows_path(path):\n    win_split = win_splitdrive(path)\n    # Note, that ntpath.splitdrive also deals with UNC paths. In that case\n    # the \"drive\" wouldn't be a windows drive letter followed by a colon\n    if win_split[0] and win_split[1] and \\\n            win_split[0].endswith(\":\") and len(win_split[0]) == 2:\n        # seems to be a windows path\n        return True\n    return False\n\n\ndef get_response_disposition_filename(s):\n    \"\"\"Given a string s as from HTTP Content-Disposition field in the response\n    return possibly present filename if any\n    \"\"\"\n    if not s:\n        return None\n    # If the response has Content-Disposition, try to get filename from it\n    cd = map(\n        lambda x: x.strip().split('=', 1) if '=' in x else [x.strip(), ''],\n        s.split(';')\n    )\n    # unify the key to be lower case and make it into a dict\n    cd = dict([[x[0].lower()] + x[1:] for x in cd])\n    if 'filename' in cd:\n        filename = cd['filename'].strip(\"\\\"'\")\n        return filename\n    return None\n\n\ndef get_url_disposition_filename(url, headers=None):\n    \"\"\"Get filename as possibly provided by the server in Content-Disposition\n    \"\"\"\n    if headers is None:\n        request = Request(url)\n        r = retry_urlopen(request)\n        # things are different in requests\n        if 'requests.' in str(r.__class__):\n            headers = r.headers\n        else:\n            headers = r.info()\n    else:\n        r = None\n    try:\n        return get_response_disposition_filename(headers.get('Content-Disposition', ''))\n    finally:\n        if r:\n            r.close()\n\n\ndef get_url_straight_filename(url, strip=None, allowdir=False):\n    \"\"\"Get file/dir name of the last path component of the URL\n\n    Parameters\n    ----------\n    strip: list, optional\n      If provided, listed names will not be considered and their\n      parent directory will be selected\n    allowdir: bool, optional\n      If url points to a \"directory\" (ends with /), empty string\n      would be returned unless allowdir is True, in which case the\n      name of the directory would be returned\n    \"\"\"\n    path = urlunquote(urlsplit(url).path)\n    path_parts = path.split('/')\n\n    if allowdir:\n        # strip empty ones\n        while len(path_parts) > 1 and not path_parts[-1]:\n            path_parts = path_parts[:-1]\n\n    if strip:\n        while path_parts and path_parts[-1] in strip:\n            path_parts = path_parts[:-1]\n\n    if path_parts:\n        return path_parts[-1]\n    else:\n        return None\n\n\ndef get_url_filename(url, headers=None, strip=None):\n    \"\"\"Get filename from the url, first consulting server about Content-Disposition\n    \"\"\"\n    filename = get_url_disposition_filename(url, headers)\n    if filename:\n        return filename\n    return get_url_straight_filename(url, strip=strip)\n\n\ndef get_url_response_stamp(url, response_info):\n    size, mtime = None, None\n    if 'Content-length' in response_info:\n        size = int(response_info['Content-length'])\n    if 'Last-modified' in response_info:\n        mtime = calendar.timegm(email.utils.parsedate(\n            response_info['Last-modified']))\n    return dict(size=size, mtime=mtime, url=url)\n\n\ndef get_tld(url):\n    \"\"\"Return top level domain from a url\n\n    Parameters\n    ----------\n    url : str\n    \"\"\"\n    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld\n    if not url.strip():\n        raise ValueError(\"Empty URL has no TLD\")\n    rec = urlsplit(url)\n    if not rec.netloc:\n        if not rec.scheme:\n            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'\n            return url.split('/', 1)[0]\n        else:\n            raise ValueError(\"It seems that only the scheme was provided without the net location/TLD\")\n    return rec.netloc\n\n\nfrom email.utils import (\n    mktime_tz,\n    parsedate_tz,\n)\n\n\ndef rfc2822_to_epoch(datestr):\n    \"\"\"Given rfc2822 date/time format, return seconds since epoch\"\"\"\n    return mktime_tz(parsedate_tz(datestr))\n\n\ndef iso8601_to_epoch(datestr):\n    \"\"\"Given ISO 8601 date/time format, return in seconds since epoch\n\n    iso8601 is used to parse properly the time zone information, which\n    can't be parsed with standard datetime strptime\n    \"\"\"\n    return calendar.timegm(iso8601.parse_date(datestr).utctimetuple())\n\n\ndef __urlopen_requests(url):\n    # XXX Workaround for now for ... broken code\n    if isinstance(url, Request):\n        url = url.get_full_url()\n    from requests import Session\n    return Session().get(url, stream=True)\n\n\ndef retry_urlopen(url, retries=3):\n    for t in range(retries):\n        try:\n            return __urlopen_requests(url)\n        except URLError as e:\n            lgr.warning(\"Received exception while reading %s: %s\" % (url, e))\n            if t == retries - 1:\n                # if we have reached allowed number of retries -- reraise\n                raise\n\n\ndef is_url_quoted(url):\n    \"\"\"Return whether URL looks being already quoted\n    \"\"\"\n    try:\n        url_ = urlunquote(url)\n        return url != url_\n    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)\n        # MIH: ValueError?\n        return False\n\n\ndef same_website(url_rec, u_rec):\n    \"\"\"Decide whether a link leads to external site\n\n    Parameters\n    ----------\n    url_rec: ParseResult\n      record for original url\n    u_rec: ParseResult\n      record for new url\n    \"\"\"\n    if isinstance(url_rec, str):\n        url_rec = urlparse(url_rec)\n    if isinstance(u_rec, str):\n        u_rec = urlparse(u_rec)\n    return (url_rec.netloc == u_rec.netloc)\n    # todo: collect more of sample cases.\n    # disabled below check while working on ratholeradio, since links\n    # could go to the parent and that is ok.  Figure out when it was\n    # desired not to go to the parent -- we might need explicit option\n    # and u_rec.path.startswith(url_rec.path)):\n\n\ndef dlurljoin(u_path, url):\n    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO\n    if url_rec.scheme:\n        # independent full url, so just return it\n        return url\n    if u_path.endswith('/'):  # should here be also a scheme use?\n        if url.startswith('/'):  # jump to the root\n            u_path_rec = urlparse(u_path)\n            return urljoin(urlunparse(\n                (u_path_rec.scheme, u_path_rec.netloc, '', '', '', '')), url)\n        else:\n            return os.path.join(u_path, url)\n    # TODO: recall where all this dirname came from and bring into the test\n    return urljoin(os.path.dirname(u_path) + '/', url)\n\n\n# TODO should it be a node maybe?\nclass SimpleURLStamper(object):\n    \"\"\"Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header\n    \"\"\"\n    def __init__(self, mode='full'):\n        self.mode = mode\n\n    def __call__(self, url):\n        # Extracted from above madness\n        # TODO: add mode alike to 'relaxed' where we would not\n        # care about content-disposition filename\n        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way\n        request = Request(url)\n\n        # No traffic compression since we do not know how to identify\n        # exactly either it has to be decompressed\n        # request.add_header('Accept-encoding', 'gzip,deflate')\n        #\n        # TODO: think about stamping etc -- we seems to be redoing\n        # what git-annex does for us already... not really\n        r = retry_urlopen(request)\n        try:\n            r_info = r.info()\n            r_stamp = get_url_response_stamp(url, r_info)\n\n            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)\n        finally:\n            r.close()\n\n\n# TODO:  make it consistent/clear at what stage % encoding/decoding happens!\n# now it is a mix!\n\n#\n# Useful functionality in requests.models\n#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance\n#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...\n#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc\n#\n\n@lru_cache(maxsize=100)\ndef _guess_ri_cls(ri):\n    \"\"\"Factory function which would determine which type of a ri a provided string is\"\"\"\n    TYPES = {\n        'url': URL,\n        'ssh':  SSHRI,\n        'file': PathRI,\n        'datalad': DataLadRI,\n        'git-transport': GitTransportRI,\n    }\n    if isinstance(ri, PurePath):\n        lgr.log(5, \"Detected file ri\")\n        return TYPES['file']\n    if is_windows_path(ri):\n        # OMG we got something from windows\n        lgr.log(5, \"Detected file ri\")\n        return TYPES['file']\n\n    # We assume that it is a URL and parse it. Depending on the result\n    # we might decide that it was something else ;)\n    fields = URL._pr_to_fields(urlparse(ri), guessing=True)\n    lgr.log(5, \"Parsed ri %s into fields %s\", ri, fields)\n    type_ = 'url'\n    # Special treatments\n    # file:///path should stay file:\n    if fields['scheme'] and fields['scheme'] not in {'file'} \\\n            and not fields['hostname']:\n        # transport::URL-or-path\n        if fields['path'].startswith(':'):  # there was ::\n            lgr.log(5, \"Assuming git transport style ri and returning\")\n            type_ = 'git-transport'\n        # dl+archive:... or just for ssh   hostname:path/p1\n        elif '+' not in fields['scheme']:\n            type_ = 'ssh'\n            lgr.log(5, \"Assuming ssh style ri, adjusted: %s\", fields)\n\n    if not fields['scheme'] and not fields['hostname']:\n        parts = _split_colon(ri)\n        # if no illegal for username@hostname characters in the first part and\n        # we either had username@hostname or multiple :-separated parts\n        if not set(parts[0]).intersection(set('/\\\\#')) and (\n                fields['path'] and\n                '@' in fields['path'] or\n                len(parts) > 1\n        ):\n            # user@host:path/sp1\n            # or host_name: (hence parts check)\n            # TODO: we need a regex to catch those really, parts check is not suff\n            type_ = 'ssh'\n        elif ri.startswith('//'):\n            # e.g. // or ///path\n            type_ = 'datalad'\n        else:\n            type_ = 'file'\n\n    if not fields['scheme'] and fields['hostname']:\n        # e.g. //a/path\n        type_ = 'datalad'\n\n    cls = TYPES[type_]\n    # just parse the ri according to regex matchint ssh \"ri\" specs\n    lgr.log(5, \"Detected %s ri\", type_)\n    return cls\n\n\nclass RI(object):\n    \"\"\"Resource Identifier - base class and a factory for URL, SSHRI, etc\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place).\n    Subclasses define specific collections of fields they care about in _FIELDS\n    class variable.\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    `RI` could be used as factory, whenever type of the resource is unknown and\n    must be guessed from the string representation.  One of the subclasses will be\n    provided as output, e.g.\n\n    >>> RI('http://example.com')\n    URL(hostname='example.com', netloc='example.com', scheme='http')\n    >>> RI('file://C:/Windows')\n    URL(hostname='c', netloc='C:', path='/Windows', scheme='file')\n    >>> RI('example.com:path')\n    SSHRI(hostname='example.com', path='path')\n    \"\"\"\n\n    # All of the subclasses will provide path\n    _FIELDS = (\n        'path',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __new__(cls, ri=None, **kwargs):\n        \"\"\"Used as a possible factory for known RI types\n\n        Returns\n        -------\n        RI\n           uninitialized RI object of appropriate class with _str\n           set to string representation if was provided\n\n        Raises\n        ------\n        ValueError\n          Whenever the RI type cannot be determined.\n        \"\"\"\n        if cls is RI and ri is not None:\n            # RI class was used as a factory\n            try:\n                cls = _guess_ri_cls(ri)\n            except Exception as e:\n                # when anything goes wrong here, ensure a homogeneous\n                # exception with a regular error\n                raise ValueError(\n                    f\"Could not determine resource identifier type for {ri!r}\"\n                ) from e\n\n        if cls is RI:\n            raise ValueError(\n                f\"Could not determine resource identifier type for {ri!r}\")\n\n        ri_obj = super(RI, cls).__new__(cls)\n        # Store internally original str\n        ri_obj._str = str(ri) if isinstance(ri, PurePath) else ri\n        return ri_obj\n\n    def __init__(self, ri=None, **fields):\n        \"\"\"\n        Parameters\n        ----------\n        ri: str, optional\n          String version of a resource specific for this class.  If you would like\n          a type of the resource be deduced, use RI(ri). Note that this value\n          will be passed to str(), so you do not have to cast it yourself.\n        **fields: dict, optional\n          The values for the fields defined in _FIELDS class variable.\n        \"\"\"\n        if ri and (bool(ri) == bool(fields)):\n            raise ValueError(\n                \"Specify either ri or breakdown from the fields, not both. \"\n                \"Got ri=%r, fields=%r\" % (ri, fields))\n\n        self._fields = self._get_blank_fields()\n        if ri is not None:\n            ri = str(ri)\n            fields = self._str_to_fields(ri)\n        self._set_from_fields(**fields)\n\n        # If was initialized from a string representation\n        if lgr.isEnabledFor(logging.DEBUG) and self._str is not None:\n            # well -- some ris might not unparse identically back\n            # strictly speaking, but let's assume they do\n            ri_ = self.as_str()\n            if ri != ri_:\n                lgr.debug(\"Parsed version of %s %r differs from original %r\",\n                          self.__class__.__name__, ri_, ri)\n\n    @classmethod\n    def _get_blank_fields(cls, **fields):\n        return OrderedDict(((f, fields.get(f, '')) for f in cls._FIELDS))\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self.as_str()\n        return self._str\n\n    @classmethod\n    def from_str(cls, ri_str):\n        obj = cls(**cls._str_to_fields(ri_str))\n        obj._str = ri_str\n        return obj\n\n    @property\n    def localpath(self):\n        # by default RIs point to remote locations\n        raise ValueError(\"%s points to remote location\" % self)\n\n    # Apparently doesn't quite play nicely with multiple inheritance for MixIn'\n    # of regexp based URLs\n    #@abstractmethod\n    #@classmethod\n    #def _str_to_fields(cls, ri_str):\n    #    raise NotImplementedError\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __bool__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields for %s are: %s\"\n                             % (unknown_fields, self.__class__, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(map_items(ensure_bytes, v))\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startswith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, RI):\n            other = RI(other)\n        return isinstance(other, self.__class__) and dict(other._fields) == dict(self._fields)\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    def __getattribute__(self, item):\n        if item.startswith('_') or item not in self._FIELDS:\n            return super(RI, self).__getattribute__(item)\n        else:\n            return self._fields[item]\n\n    def __setattr__(self, item, value):\n        if item.startswith('_') or item not in self._FIELDS:\n            super(RI, self).__setattr__(item, value)\n        else:\n            self._fields[item] = value\n            self._str = None\n\n\nclass URL(RI):\n    \"\"\"Universal resource locator\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    \"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'scheme',\n        'netloc',\n        'username',\n        'password',\n        'hostname', 'port',\n        'query',\n        'fragment',\n    )\n\n    # Only interpreted on Windows. If set to `True`, UNC-names encoded in\n    # file-URLs, e.g. \"file://server/share/path\", would be considered local\n    # and mapped onto \"\\\\server\\share\\path\". If set to `False`, UNC-names\n    # encoded in file-URLs would not be considered local and cannot be resolved\n    # to a local path.\n    support_unc = False\n\n    def as_str(self):\n        \"\"\"Render URL as a string\"\"\"\n        return urlunparse(self.to_pr())\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        fields = URL._pr_to_fields(urlparse(url_str))\n        fields['path'] = urlunquote(fields['path'])\n        return fields\n\n    def to_pr(self):\n        \"\"\"Convert URL to urlparse.ParseResults namedtuple\"\"\"\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            if fields['hostname'].count(':') >= 2:\n                # ipv6 -- need to enclose in []\n                netloc = '[%s]:%s' % (netloc, fields['port'])\n            else:\n                netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n        # We need to quote the path\n        pr_fields['path'] = urlquote(pr_fields['path'])\n        # TODO: figure out what to do with query/fragment... one step at a time\n        return ParseResult(**pr_fields)\n\n    @classmethod\n    def _pr_to_fields(cls, pr, guessing=False):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n        if pr.params:\n            (lgr.debug if guessing else lgr.warning)(\n                \"ParseResults contains params %s, which will be ignored\",\n                repr(pr.params),)\n\n        hostname_port = pr.netloc.split('@')[-1]\n        is_ipv6 = hostname_port.count(':') >= 2\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        if is_ipv6:\n            rem = re.match(r'\\[(?P<hostname>.*)\\]:(?P<port>\\d+)', hostname_port)\n            if rem:\n                hostname, port = rem.groups()\n                port = int(port)\n            else:\n                hostname, port = hostname_port, ''\n\n            def _getattr(pr, f):\n                \"\"\"Helper for custom handling in case of ipv6 addresses which blows\n                stock ParseResults logic\"\"\"\n                if f == 'port':\n                    # for now not supported at all, so\n                    return port\n                elif f == 'hostname':\n                    return hostname\n                else:\n                    return getattr(pr, f)\n        else:\n            _getattr = getattr\n\n        return {f: (_getattr(pr, f) or '') for f in cls._FIELDS}\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = map_items(ensure_unicode, OrderedDict(parse_qsl(s, 1)))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)\n\n    def _windows_local_path(self,\n                            support_unc: bool = False\n                            ) -> str:\n        \"\"\"Convert the URL to a local path on windows, supports UNC and git-annex\"\"\"\n\n        # RFC1738 and RFC3986 both forbid unescaped backslash characters in\n        # URLs, and therefore also in the path-component of file:-URLs. We\n        # assume here that any backslash present in a file-URL is a relict of a\n        # verbatim copy of a Windows-style path.\n        unified_path = self.path.replace('\\\\', '/')\n        local_path = url2pathname(unified_path)\n\n        # We support UNC notation, and the \"special\" git-annex drive encoding\n        # scheme, i.e. netloc is the drive letter plus a colon.\n        # NB, this if clause will not evaluate to True, because our caller\n        # filters out net locations with\n        if self.netloc:\n            if re.match('^[a-zA-Z]:$', self.netloc):\n                # This is the git-annex case, i.e. drive spec in netloc\n                return self.netloc + local_path\n            if support_unc:\n                return '\\\\\\\\' + self.netloc + local_path\n            raise ValueError(\"Unsupported file: URL: {self}\")\n        return local_path\n\n    @property\n    def localpath(self):\n        if self.scheme != 'file':\n            raise ValueError(\n                \"Non 'file://' URL cannot be resolved to a local path\")\n\n        # If there is a hostname, we might want to convert to UNC, unless the\n        # hostname has the form of a 'Windows drive letter' on windows\n        hostname = self.hostname\n        if not (hostname in (None, '', 'localhost', '::1')\n                or hostname.startswith('127.')\n                or re.match('^[a-zA-Z]:$', self.netloc)):\n            if on_windows:\n                return self._windows_local_path(support_unc=self.support_unc)\n            raise ValueError(\"file:// URL does not point to 'localhost'\")\n\n        if on_windows:\n            return self._windows_local_path(support_unc=True)\n\n        return url2pathname(self.path)\n\n\nclass PathRI(RI):\n    \"\"\"RI pointing to a (local) file/directory\"\"\"\n    def as_str(self):\n        return self.path\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        # str() to be compatible with pathlib objects\n        return dict(path=str(url_str))\n\n    @property\n    def localpath(self):\n        return str(Path(self.path))\n\n    @property\n    def posixpath(self):\n        if is_windows_path(self.path):\n            win_split = win_splitdrive(self.path)\n            return \"/\" + win_split[0][0] + win_split[1].replace('\\\\', '/')\n        else:\n            return self.path\n\n\nclass RegexBasedURLMixin(object):\n    \"\"\"Base class for URLs which we could simple parse using regular expressions\"\"\"\n\n    _REGEX = None\n\n    # not used ATM but possible ;)\n    # @classmethod\n    # def is_str_matches(cls, url_str):\n    #     return bool(cls._REGEX.match(url_str))\n\n    @classmethod\n    def _str_to_fields(cls, url_str):\n        re_match = cls._REGEX.match(url_str)\n        if not re_match:\n            # TODO: custom error?\n            raise ValueError(\n                \"Cannot handle URL '%s': categorized as %r, but does not match syntax.%s\"\n                % (url_str,\n                   cls,\n                   \" Did you intent to use '///'?\" if url_str.startswith('//') else '')\n            )\n        fields = cls._get_blank_fields()\n        fields.update({k: v for k, v in re_match.groupdict().items() if v})\n        cls._normalize_fields(fields)\n        return fields\n\n    @classmethod\n    def _normalize_fields(self, fields):\n        \"\"\"Helper to be ran if any of the fields need to be normalized after parsing\"\"\"\n        pass\n\n\nclass SSHRI(RI, RegexBasedURLMixin):\n    \"\"\"RI pointing to a remote location reachable via SSH\"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'username',\n        'hostname',\n        'port',\n    )\n\n    _REGEX = re.compile(r'((?P<username>\\S*)@)?(?P<hostname>[^#/\\\\:]+)(\\:(?P<path>.*))?$')\n\n    @classmethod\n    def _normalize_fields(cls, fields):\n        if fields['path'] and fields['path'].startswith('//'):\n            # Let's normalize for now to avoid multiple leading slashes\n            fields['path'] = '/' + fields['path'].lstrip('/')\n        # escape path so we have direct representation of the path to work with\n        fields['path'] = unescape_ssh_path(fields['path'])\n\n    def as_str(self, escape=False):\n        fields = self.fields  # copy so we could escape symbols\n        url_fmt = '{hostname}'\n        if fields['username']:\n            url_fmt = \"{username}@\" + url_fmt\n        if fields['path']:\n            url_fmt += ':{path}'\n        if escape:\n            fields['path'] = escape_ssh_path(fields['path'])\n        return url_fmt.format(**fields)\n\n    # TODO:\n    # we can \"support\" localhost:path as localpaths\n\n\nclass DataLadRI(RI, RegexBasedURLMixin):\n    \"\"\"RI pointing to datasets within default DataLad super-dataset\"\"\"\n\n    _FIELDS = RI._FIELDS + (\n        'remote',\n    )\n\n    # For now or forever we don't deal with any fragments or other special stuff\n    _REGEX = re.compile(r'//(?P<remote>[^\\s/]*)/(?P<path>.*)$')\n\n    # do they need to be normalized??? losing track ...\n\n    def as_str(self):\n        return \"//{remote}/{path}\".format(**self._fields)\n\n    def as_git_url(self):\n        \"\"\"Dereference /// into original URLs which could be used by git for cloning\n\n        Returns\n        -------\n        str\n          URL string to reference the DataLadRI from its /// form\n        \"\"\"\n        if self.remote:\n            raise NotImplementedError(\"not supported ATM to reference additional remotes\")\n        return \"{}{}\".format(consts.DATASETS_TOPURL, urlquote(self.path))\n\n\nclass GitTransportRI(RI, RegexBasedURLMixin):\n    \"\"\"RI for some other RI with git transport prefix\"\"\"\n\n    # TODO: check how crticial to \"inherit\" RI._FIELDS asking to provide path\n    _FIELDS = RI._FIELDS + (\n        'transport',\n        'RI',\n    )\n\n    # Due to poor design, `ri` argument already present in various\n    # places intermixed with **kwargs treatment. So we will use RI\n    # here instead of ri.\n    _REGEX = re.compile(r'(?P<transport>[A-Za-z0-9][A-Za-z0-9+.-]*)::(?P<RI>.*)$')\n\n    def as_str(self):\n        return '{self.transport}::{self.RI}'.format(self=self)\n\n\ndef _split_colon(s, maxsplit=1):\n    \"\"\"Split on unescaped colon\"\"\"\n    return re.compile(r'(?<!\\\\):').split(s, maxsplit=maxsplit)\n\n# \\ should be first to deal with\n_SSH_ESCAPED_CHARACTERS = '\\\\#&;`|*?~<>^()[]{}$\\'\" '\n\n\n# TODO: RF using re.sub\ndef escape_ssh_path(path):\n    \"\"\"Escape all special characters present in the path\"\"\"\n    for c in _SSH_ESCAPED_CHARACTERS:\n        if c in path:\n            path = path.replace(c, '\\\\' + c)\n    return path\n\n\ndef unescape_ssh_path(path):\n    \"\"\"Un-escape all special characters present in the path\"\"\"\n    for c in _SSH_ESCAPED_CHARACTERS[::-1]:\n        if c in path:\n            path = path.replace('\\\\' + c, c)\n    return path\n\n\ndef parse_url_opts(url):\n    \"\"\"Given a string with url-style query, split into content before # and options as dict\"\"\"\n    url = URL(url)\n    # we need to filter out query and fragment to get the base url\n    fields = url.fields\n    fields.pop('query')\n    fields.pop('fragment')\n    opts = url.query_dict\n    return str(URL(**fields)), opts\n\n\n# TODO: should we just define URL.good_for_git or smth like that? ;)\n# although git also understands regular paths\ndef is_url(ri):\n    \"\"\"Returns whether argument is a resource identifier what datalad should treat as a URL\n\n    This includes ssh \"urls\" which git understands.\n\n    Parameters\n    ----------\n    ri : str or RI\n      The resource identifier (as a string or RI) to \"analyze\"\n    \"\"\"\n    if not isinstance(ri, RI):\n        try:\n            ri = RI(ri)\n        except:  # MIH: MemoryError?\n            return False\n    return isinstance(ri, (URL, SSHRI))\n\n\n# TODO: RF to remove duplication\ndef is_datalad_compat_ri(ri):\n    \"\"\"Returns whether argument is a resource identifier what datalad should treat as a URL\n\n    including its own DataLadRI\n    \"\"\"\n    if not isinstance(ri, RI):\n        try:\n            ri = RI(ri)\n        except:  # MIH: MemoryError?\n            return False\n    return isinstance(ri, (URL, SSHRI, DataLadRI))\n\n\n# TODO: better name? additionally may be move to SSHRI.is_valid() or sth.\ndef is_ssh(ri):\n    \"\"\"helper to determine, whether `ri` requires an SSH connection\n\n    Parameters\n    ----------\n    ri: str or RI\n\n    Returns\n    -------\n    bool\n    \"\"\"\n\n    # not exactly fitting the doc, but we actually can deal not necessarily with\n    # string or RI only, but with everything RI itself can deal with:\n    _ri = RI(ri) if not isinstance(ri, RI) else ri\n\n    return isinstance(_ri, SSHRI) \\\n        or (isinstance(_ri, URL) and _ri.scheme == 'ssh')\n\n\ndef get_local_file_url(fname: str,\n                       compatibility: str = 'git-annex',\n                       allow_relative_path: bool = True\n                       ) -> str:\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    compatibility : str, optional\n        This parameter is only interpreted on Windows systems. If set to\n        anything else than 'git', the anchor, e.g. `C:` of `fname` will be put\n        into the `file-auth` part, i.e. network location, defined in RFC 8089.\n        This option is mainly used to support git-annex specific encoding of\n        Windows paths.\n    allow_relative_path: bool, optional\n        Allow `fname` to be a relative path. The path will be converted to an\n        absolute path, by using the current directory as path prefix.\n    \"\"\"\n    path_obj = Path(fname)\n    if not path_obj.is_absolute():\n        if allow_relative_path:\n            fname = str(Path(os.getcwd()) / path_obj)\n        else:\n            raise ValueError('cannot create file-URL with relative path')\n\n    url_path = local_path2url_path(fname, auto_resolve=False)\n    if on_windows and compatibility != \"git\":\n        # Work around the way in which git-annex interprets file URLs on\n        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`\n        # into the network location component of the resulting URL.\n        return \"file:/\" + url_path\n\n    result = \"file://\" + url_path\n    if result != Path(fname).as_uri():\n        lgr.warning(\n            f\"get_local_file_url({fname}, {compatibility}, \"\n            f\"{allow_relative_path}) returned {result} instead of \"\n            f\"{Path(fname).as_uri()}\")\n    return result\n\n\ndef get_url_cache_filename(url, name=None):\n    \"\"\"Return a filename where to cache online doc from a url\"\"\"\n    if not name:\n        name = \"misc\"\n    cache_dir = opj(cfg.obtain('datalad.locations.cache'), name)\n    doc_fname = opj(\n        cache_dir,\n        '{}-{}.p{}'.format(\n            urlsplit(url).netloc,\n            md5(url.encode('utf-8')).hexdigest(),\n            pickle.HIGHEST_PROTOCOL)\n    )\n    return doc_fname\n\n\ndef get_cached_url_content(url, name=None, fetcher=None, maxage=None):\n    \"\"\"Loader of a document from a url, which caches loaded instance on disk\n\n    Doesn't do anything smart about http headers etc which could provide\n    information for cache/proxy servers for how long to retain etc\n\n    TODO: theoretically it is not network specific at all -- and just a memoize\n    pattern, but may be some time we would make it treat headers etc correctly.\n    And ATM would support any URL we support via providers/downloaders\n\n    Parameters\n    ----------\n    fetcher: callable, optional\n       Function to call with url if needed to be refetched\n    maxage: float, optional\n       Age in days to retain valid for.  <0 - would retain forever.  If None -\n       would consult the config, 0 - would force to reload\n    \"\"\"\n    doc_fname = get_url_cache_filename(url, name)\n    if maxage is None:\n        maxage = float(cfg.get('datalad.locations.cache-maxage'))\n\n    doc = None\n    if os.path.exists(doc_fname) and maxage != 0:\n\n        fage = (time.time() - os.stat(doc_fname).st_mtime)/(24. * 3600)\n        if maxage < 0 or fage < maxage:\n            try:\n                lgr.debug(\"use cached request result to '%s' from %s\", url, doc_fname)\n                doc = pickle.load(open(doc_fname, 'rb'))\n            except Exception as e:  # it is OK to ignore any error and fall back on the true source\n                lgr.warning(\n                    \"cannot load cache from '%s', fall back to download: %s\",\n                    doc_fname, CapturedException(e))\n\n    if doc is None:\n        if fetcher is None:\n            from datalad.downloaders.providers import Providers\n            providers = Providers.from_config_files()\n            fetcher = providers.fetch\n\n        doc = fetcher(url)\n        ensure_dir(dirname(doc_fname))\n        # use pickle to store the entire request result dict\n        pickle.dump(doc, open(doc_fname, 'wb'))\n        lgr.debug(\"stored result of request to '%s' in %s\", url, doc_fname)\n    return doc\n\n\ndef download_url(url, dest=None, overwrite=False):\n    \"\"\"Download a file from a URL\n\n    Supports and honors any DataLad \"downloader/provider\" configuration.\n\n    Parameters\n    ----------\n    url: str\n      Source URL to download from.\n    dest: Path-like or None\n      Destination file name (file must not exist), or name of a target\n      directory (must exists, and filename must be derivable from `url`).\n      If None, the downloaded content will be returned as a string.\n    overwrite: bool\n      Force overwriting an existing destination file.\n\n    Returns\n    -------\n    str\n      Path of the downloaded file, or URL content if `dest` is None.\n\n    Raises\n    ------\n    DownloadError\n      If `dest` already exists and is a file, or if `dest` is a directory\n      and no filename could be determined from `url`, or if no file was\n      found at the given `url`.\n    \"\"\"\n    from datalad.downloaders.providers import Providers\n    providers = Providers.from_config_files()\n    if dest:\n        return providers.download(url, path=str(dest), overwrite=overwrite)\n    else:\n        return providers.fetch(url)\n\n\ndef local_path2url_path(local_path: str,\n                        auto_resolve: bool = False\n                        ) -> str:\n    \"\"\"Convert a local path into a URL path component\"\"\"\n    if not local_path:\n        if not auto_resolve:\n            raise ValueError(\"cannot convert empty local path to URL path\")\n        local_path = os.getcwd()\n\n    url = urlparse(Path(local_path).as_uri())\n    if url.netloc:\n        raise ValueError(\n            f\"cannot convert remote path to an URL path: {local_path}\")\n    return url.path\n\n\ndef url_path2local_path(url_path: str | PurePosixPath,\n                        make_absolute: bool = True,\n                        ) -> str | Path:\n\n    if isinstance(url_path, PurePosixPath):\n        return_path = True\n        url_path = str(url_path)\n    else:\n        return_path = False\n\n    if not url_path or not url_path.startswith(\"/\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # path must begin with a slash.\n        raise ValueError(\n            f\"url path does not start with '/': {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    if url_path.startswith(\"//\"):\n        # We expect a 'path-absolute' as defined in RFC 3986, therefore the\n        # first segment must not be empty, i.e. the path must not start with\n        # two or more slashes.\n        raise ValueError(\n            f\"url path has empty first segment: {url_path}, and is therefore \"\n            f\"not an absolute-path as defined in RFC 8089\")\n\n    return (\n        Path(url2pathname(url_path))\n        if return_path\n        else url2pathname(url_path)\n    )\n\n\ndef quote_path(path: str, safe: str = \"/\") -> str:\n    \"\"\"quote the path component of a URL, takes OS specifics into account\"\"\"\n    if on_windows:\n        if re.match(\"^/[a-zA-Z]:/\", path):\n            return path[:3] + quote(path[3:], safe=safe)\n    return quote(path, safe=safe)\n\n\nlgr.log(5, \"Done importing support.network\")\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]

**Title:** `get_local_file_url` Generates Different URI Than `Path.as_uri()` for Absolute Paths

**Description:**
When converting an absolute file path to a URL using `get_local_file_url`, the resulting URI does not match the one produced by `Path.as_uri()`. This inconsistency triggers a warning and may lead to unexpected behavior when handling file URLs.

**Example Code:**
```python
from datalad.support.network import get_local_file_url
from pathlib import Path

absolute_path = "/absolute/path/to/file"
generated_url = get_local_file_url(absolute_path)
expected_url = Path(absolute_path).as_uri()

print(generated_url)  # Outputs: file:///absolute/path/to/file
print(expected_url)   # Outputs: file:///absolute/path/to/file
```

**Expected Behavior:**
Both `generated_url` and `expected_url` should be identical, resulting in the same URI without any warnings.

**Actual Behavior:**
`get_local_file_url` produces a URI that differs from `Path.as_uri()`, leading to a warning:
```
WARNING: get_local_file_url(/absolute/path/to/file, ...) returned file:///absolute/path/to/file instead of file:///absolute/path/to/file
```

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/network.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 et:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
from __future__ import annotations

import logging

lgr = logging.getLogger('datalad.network')

lgr.log(5, "Importing support.network")
import calendar
import email.utils
import os
import pickle
import re
import time
from collections import OrderedDict
from hashlib import md5
from ntpath import splitdrive as win_splitdrive
from os.path import dirname
from os.path import join as opj
from pathlib import PurePosixPath
from urllib.error import URLError
from urllib.parse import (
    ParseResult,
    parse_qsl,
)
from urllib.parse import quote as urlquote
from urllib.parse import unquote as urlunquote
from urllib.parse import (
    quote,
    urlencode,
    urljoin,
    urlparse,
    urlsplit,
    urlunparse,
)
from urllib.request import (
    Request,
    url2pathname,
)

import iso8601

from datalad import (
    cfg,
    consts,
)
from datalad.support.cache import lru_cache
from datalad.support.exceptions import CapturedException
from datalad.utils import (
    Path,
    PurePath,
    ensure_bytes,
    ensure_dir,
    ensure_unicode,
    map_items,
    on_windows,
)

# !!! Lazily import requests where needed -- needs 30ms or so
# import requests


def local_path_representation(path: str) -> str:
    """Return an OS-specific representation of a Posix-style path

    With a posix path in the form of "a/b" this function will return "a/b" on
    Unix-like operating systems and "a\\b" on Windows-style operating systems.
    """
    return str(Path(path))


def local_url_path_representation(url_path: str) -> str:
    """Return an OS-specific representation of the path component in a file:-URL

    With a path component like "/c:/Windows" (i.e. from a URL that reads
    "file:///c:/Windows"), this function will return "/c:/Windows" on a
    Unix-like operating systems and "C:\\Windows" on Windows-like operating
    systems.
    """
    return url2pathname(quote(url_path))


def local_path_from_url(url: str) -> str:
    """Parse the url and extract an OS-specific local path representation"""
    return local_url_path_representation(urlparse(url).path)


def is_windows_path(path):
    win_split = win_splitdrive(path)
    # Note, that ntpath.splitdrive also deals with UNC paths. In that case
    # the "drive" wouldn't be a windows drive letter followed by a colon
    if win_split[0] and win_split[1] and \
            win_split[0].endswith(":") and len(win_split[0]) == 2:
        # seems to be a windows path
        return True
    return False


def get_response_disposition_filename(s):
    """Given a string s as from HTTP Content-Disposition field in the response
    return possibly present filename if any
    """
    if not s:
        return None
    # If the response has Content-Disposition, try to get filename from it
    cd = map(
        lambda x: x.strip().split('=', 1) if '=' in x else [x.strip(), ''],
        s.split(';')
    )
    # unify the key to be lower case and make it into a dict
    cd = dict([[x[0].lower()] + x[1:] for x in cd])
    if 'filename' in cd:
        filename = cd['filename'].strip("\"'")
        return filename
    return None


def get_url_disposition_filename(url, headers=None):
    """Get filename as possibly provided by the server in Content-Disposition
    """
    if headers is None:
        request = Request(url)
        r = retry_urlopen(request)
        # things are different in requests
        if 'requests.' in str(r.__class__):
            headers = r.headers
        else:
            headers = r.info()
    else:
        r = None
    try:
        return get_response_disposition_filename(headers.get('Content-Disposition', ''))
    finally:
        if r:
            r.close()


def get_url_straight_filename(url, strip=None, allowdir=False):
    """Get file/dir name of the last path component of the URL

    Parameters
    ----------
    strip: list, optional
      If provided, listed names will not be considered and their
      parent directory will be selected
    allowdir: bool, optional
      If url points to a "directory" (ends with /), empty string
      would be returned unless allowdir is True, in which case the
      name of the directory would be returned
    """
    path = urlunquote(urlsplit(url).path)
    path_parts = path.split('/')

    if allowdir:
        # strip empty ones
        while len(path_parts) > 1 and not path_parts[-1]:
            path_parts = path_parts[:-1]

    if strip:
        while path_parts and path_parts[-1] in strip:
            path_parts = path_parts[:-1]

    if path_parts:
        return path_parts[-1]
    else:
        return None


def get_url_filename(url, headers=None, strip=None):
    """Get filename from the url, first consulting server about Content-Disposition
    """
    filename = get_url_disposition_filename(url, headers)
    if filename:
        return filename
    return get_url_straight_filename(url, strip=strip)


def get_url_response_stamp(url, response_info):
    size, mtime = None, None
    if 'Content-length' in response_info:
        size = int(response_info['Content-length'])
    if 'Last-modified' in response_info:
        mtime = calendar.timegm(email.utils.parsedate(
            response_info['Last-modified']))
    return dict(size=size, mtime=mtime, url=url)


def get_tld(url):
    """Return top level domain from a url

    Parameters
    ----------
    url : str
    """
    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld
    if not url.strip():
        raise ValueError("Empty URL has no TLD")
    rec = urlsplit(url)
    if not rec.netloc:
        if not rec.scheme:
            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'
            return url.split('/', 1)[0]
        else:
            raise ValueError("It seems that only the scheme was provided without the net location/TLD")
    return rec.netloc


from email.utils import (
    mktime_tz,
    parsedate_tz,
)


def rfc2822_to_epoch(datestr):
    """Given rfc2822 date/time format, return seconds since epoch"""
    return mktime_tz(parsedate_tz(datestr))


def iso8601_to_epoch(datestr):
    """Given ISO 8601 date/time format, return in seconds since epoch

    iso8601 is used to parse properly the time zone information, which
    can't be parsed with standard datetime strptime
    """
    return calendar.timegm(iso8601.parse_date(datestr).utctimetuple())


def __urlopen_requests(url):
    # XXX Workaround for now for ... broken code
    if isinstance(url, Request):
        url = url.get_full_url()
    from requests import Session
    return Session().get(url, stream=True)


def retry_urlopen(url, retries=3):
    for t in range(retries):
        try:
            return __urlopen_requests(url)
        except URLError as e:
            lgr.warning("Received exception while reading %s: %s" % (url, e))
            if t == retries - 1:
                # if we have reached allowed number of retries -- reraise
                raise


def is_url_quoted(url):
    """Return whether URL looks being already quoted
    """
    try:
        url_ = urlunquote(url)
        return url != url_
    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)
        # MIH: ValueError?
        return False


def same_website(url_rec, u_rec):
    """Decide whether a link leads to external site

    Parameters
    ----------
    url_rec: ParseResult
      record for original url
    u_rec: ParseResult
      record for new url
    """
    if isinstance(url_rec, str):
        url_rec = urlparse(url_rec)
    if isinstance(u_rec, str):
        u_rec = urlparse(u_rec)
    return (url_rec.netloc == u_rec.netloc)
    # todo: collect more of sample cases.
    # disabled below check while working on ratholeradio, since links
    # could go to the parent and that is ok.  Figure out when it was
    # desired not to go to the parent -- we might need explicit option
    # and u_rec.path.startswith(url_rec.path)):


def dlurljoin(u_path, url):
    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO
    if url_rec.scheme:
        # independent full url, so just return it
        return url
    if u_path.endswith('/'):  # should here be also a scheme use?
        if url.startswith('/'):  # jump to the root
            u_path_rec = urlparse(u_path)
            return urljoin(urlunparse(
                (u_path_rec.scheme, u_path_rec.netloc, '', '', '', '')), url)
        else:
            return os.path.join(u_path, url)
    # TODO: recall where all this dirname came from and bring into the test
    return urljoin(os.path.dirname(u_path) + '/', url)


# TODO should it be a node maybe?
class SimpleURLStamper(object):
    """Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header
    """
    def __init__(self, mode='full'):
        self.mode = mode

    def __call__(self, url):
        # Extracted from above madness
        # TODO: add mode alike to 'relaxed' where we would not
        # care about content-disposition filename
        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way
        request = Request(url)

        # No traffic compression since we do not know how to identify
        # exactly either it has to be decompressed
        # request.add_header('Accept-encoding', 'gzip,deflate')
        #
        # TODO: think about stamping etc -- we seems to be redoing
        # what git-annex does for us already... not really
        r = retry_urlopen(request)
        try:
            r_info = r.info()
            r_stamp = get_url_response_stamp(url, r_info)

            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)
        finally:
            r.close()


# TODO:  make it consistent/clear at what stage % encoding/decoding happens!
# now it is a mix!

#
# Useful functionality in requests.models
#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance
#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...
#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc
#

@lru_cache(maxsize=100)
def _guess_ri_cls(ri):
    """Factory function which would determine which type of a ri a provided string is"""
    TYPES = {
        'url': URL,
        'ssh':  SSHRI,
        'file': PathRI,
        'datalad': DataLadRI,
        'git-transport': GitTransportRI,
    }
    if isinstance(ri, PurePath):
        lgr.log(5, "Detected file ri")
        return TYPES['file']
    if is_windows_path(ri):
        # OMG we got something from windows
        lgr.log(5, "Detected file ri")
        return TYPES['file']

    # We assume that it is a URL and parse it. Depending on the result
    # we might decide that it was something else ;)
    fields = URL._pr_to_fields(urlparse(ri), guessing=True)
    lgr.log(5, "Parsed ri %s into fields %s", ri, fields)
    type_ = 'url'
    # Special treatments
    # file:///path should stay file:
    if fields['scheme'] and fields['scheme'] not in {'file'} \
            and not fields['hostname']:
        # transport::URL-or-path
        if fields['path'].startswith(':'):  # there was ::
            lgr.log(5, "Assuming git transport style ri and returning")
            type_ = 'git-transport'
        # dl+archive:... or just for ssh   hostname:path/p1
        elif '+' not in fields['scheme']:
            type_ = 'ssh'
            lgr.log(5, "Assuming ssh style ri, adjusted: %s", fields)

    if not fields['scheme'] and not fields['hostname']:
        parts = _split_colon(ri)
        # if no illegal for username@hostname characters in the first part and
        # we either had username@hostname or multiple :-separated parts
        if not set(parts[0]).intersection(set('/\\#')) and (
                fields['path'] and
                '@' in fields['path'] or
                len(parts) > 1
        ):
            # user@host:path/sp1
            # or host_name: (hence parts check)
            # TODO: we need a regex to catch those really, parts check is not suff
            type_ = 'ssh'
        elif ri.startswith('//'):
            # e.g. // or ///path
            type_ = 'datalad'
        else:
            type_ = 'file'

    if not fields['scheme'] and fields['hostname']:
        # e.g. //a/path
        type_ = 'datalad'

    cls = TYPES[type_]
    # just parse the ri according to regex matchint ssh "ri" specs
    lgr.log(5, "Detected %s ri", type_)
    return cls


class RI(object):
    """Resource Identifier - base class and a factory for URL, SSHRI, etc

    Intended to be a R/O object (i.e. no fields should be changed in-place).
    Subclasses define specific collections of fields they care about in _FIELDS
    class variable.
    The idea is that this class should help to break apart a URL, while being
    able to rebuild itself into a string representation for reuse

    `RI` could be used as factory, whenever type of the resource is unknown and
    must be guessed from the string representation.  One of the subclasses will be
    provided as output, e.g.

    >>> RI('http://example.com')
    URL(hostname='example.com', netloc='example.com', scheme='http')
    >>> RI('file://C:/Windows')
    URL(hostname='c', netloc='C:', path='/Windows', scheme='file')
    >>> RI('example.com:path')
    SSHRI(hostname='example.com', path='path')
    """

    # All of the subclasses will provide path
    _FIELDS = (
        'path',
    )

    __slots__ = _FIELDS + ('_fields', '_str')

    def __new__(cls, ri=None, **kwargs):
        """Used as a possible factory for known RI types

        Returns
        -------
        RI
           uninitialized RI object of appropriate class with _str
           set to string representation if was provided

        Raises
        ------
        ValueError
          Whenever the RI type cannot be determined.
        """
        if cls is RI and ri is not None:
            # RI class was used as a factory
            try:
                cls = _guess_ri_cls(ri)
            except Exception as e:
                # when anything goes wrong here, ensure a homogeneous
                # exception with a regular error
                raise ValueError(
                    f"Could not determine resource identifier type for {ri!r}"
                ) from e

        if cls is RI:
            raise ValueError(
                f"Could not determine resource identifier type for {ri!r}")

        ri_obj = super(RI, cls).__new__(cls)
        # Store internally original str
        ri_obj._str = str(ri) if isinstance(ri, PurePath) else ri
        return ri_obj

    def __init__(self, ri=None, **fields):
        """
        Parameters
        ----------
        ri: str, optional
          String version of a resource specific for this class.  If you would like
          a type of the resource be deduced, use RI(ri). Note that this value
          will be passed to str(), so you do not have to cast it yourself.
        **fields: dict, optional
          The values for the fields defined in _FIELDS class variable.
        """
        if ri and (bool(ri) == bool(fields)):
            raise ValueError(
                "Specify either ri or breakdown from the fields, not both. "
                "Got ri=%r, fields=%r" % (ri, fields))

        self._fields = self._get_blank_fields()
        if ri is not None:
            ri = str(ri)
            fields = self._str_to_fields(ri)
        self._set_from_fields(**fields)

        # If was initialized from a string representation
        if lgr.isEnabledFor(logging.DEBUG) and self._str is not None:
            # well -- some ris might not unparse identically back
            # strictly speaking, but let's assume they do
            ri_ = self.as_str()
            if ri != ri_:
                lgr.debug("Parsed version of %s %r differs from original %r",
                          self.__class__.__name__, ri_, ri)

    @classmethod
    def _get_blank_fields(cls, **fields):
        return OrderedDict(((f, fields.get(f, '')) for f in cls._FIELDS))

    @property
    def fields(self):
        """Returns shallow copy of fields to ease manipulations"""
        return self._fields.copy()

    def __repr__(self):
        # since auto_repr doesn't support "non-0" values atm
        return "%s(%s)" % (
            self.__class__.__name__,
            ", ".join(["%s=%r" % (k, v)
                       for k, v in sorted(self._fields.items())
                       if v]))

    # Lazily evaluated if _str was not set
    def __str__(self):
        if self._str is None:
            self._str = self.as_str()
        return self._str

    @classmethod
    def from_str(cls, ri_str):
        obj = cls(**cls._str_to_fields(ri_str))
        obj._str = ri_str
        return obj

    @property
    def localpath(self):
        # by default RIs point to remote locations
        raise ValueError("%s points to remote location" % self)

    # Apparently doesn't quite play nicely with multiple inheritance for MixIn'
    # of regexp based URLs
    #@abstractmethod
    #@classmethod
    #def _str_to_fields(cls, ri_str):
    #    raise NotImplementedError

    #
    # If any field is specified, URL is not considered 'False', i.e.
    # non-existing, although may be we could/shout omit having only
    # scheme or port specified since it doesn't point to any useful
    # location
    #

    def __bool__(self):
        fields = self._fields
        return any(fields.values())

    #
    # Helpers to deal with internal structures and conversions
    #

    def _set_from_fields(self, **fields):
        unknown_fields = set(fields).difference(self._FIELDS)
        if unknown_fields:
            raise ValueError("Do not know about %s. Known fields for %s are: %s"
                             % (unknown_fields, self.__class__, self._FIELDS))

        # encode dicts for query or fragment into
        for f in {'query', 'fragment'}:
            v = fields.get(f)
            if isinstance(v, dict):

                ev = urlencode(map_items(ensure_bytes, v))
                # / is reserved char within query
                if f == 'fragment' and '%2F' not in str(v):
                    # but seems to be ok'ish within the fragment which is
                    # the last element of URI and anyways used only by the
                    # client (i.e. by us here if used to compose the URL)
                    # so let's return / back for clarity if there were no
                    # awkward %2F to startswith
                    ev = ev.replace('%2F', '/')
                fields[f] = ev

        self._fields.update(fields)

    #
    # Quick comparators
    #

    def __eq__(self, other):
        if not isinstance(other, RI):
            other = RI(other)
        return isinstance(other, self.__class__) and dict(other._fields) == dict(self._fields)

    def __ne__(self, other):
        return not (self == other)

    def __getattribute__(self, item):
        if item.startswith('_') or item not in self._FIELDS:
            return super(RI, self).__getattribute__(item)
        else:
            return self._fields[item]

    def __setattr__(self, item, value):
        if item.startswith('_') or item not in self._FIELDS:
            super(RI, self).__setattr__(item, value)
        else:
            self._fields[item] = value
            self._str = None


class URL(RI):
    """Universal resource locator

    Although largely decorating urlparse.ParseResult, it
    - doesn't mandate providing all parts of the URL
    - doesn't require netloc but rather asks for separate username, password, and hostname
    """

    _FIELDS = RI._FIELDS + (
        'scheme',
        'netloc',
        'username',
        'password',
        'hostname', 'port',
        'query',
        'fragment',
    )

    # Only interpreted on Windows. If set to `True`, UNC-names encoded in
    # file-URLs, e.g. "file://server/share/path", would be considered local
    # and mapped onto "\\server\share\path". If set to `False`, UNC-names
    # encoded in file-URLs would not be considered local and cannot be resolved
    # to a local path.
    support_unc = False

    def as_str(self):
        """Render URL as a string"""
        return urlunparse(self.to_pr())

    @classmethod
    def _str_to_fields(cls, url_str):
        fields = URL._pr_to_fields(urlparse(url_str))
        fields['path'] = urlunquote(fields['path'])
        return fields

    def to_pr(self):
        """Convert URL to urlparse.ParseResults namedtuple"""
        return self._fields_to_pr(self._fields)

    @classmethod
    def _fields_to_pr(cls, fields):
        """Recompose back fields dict to ParseResult"""
        netloc = fields['username'] or ''
        if fields['password']:
            netloc += ':' + fields['password']
        if netloc:
            netloc += '@'
        netloc += fields['hostname']
        if fields['port']:
            if fields['hostname'].count(':') >= 2:
                # ipv6 -- need to enclose in []
                netloc = '[%s]:%s' % (netloc, fields['port'])
            else:
                netloc += ':%s' % fields['port']

        pr_fields = {
            f: fields[f]
            for f in cls._FIELDS
            if f not in ('hostname', 'password', 'username', 'port')
        }
        pr_fields['netloc'] = netloc
        pr_fields['params'] = ''
        # We need to quote the path
        pr_fields['path'] = urlquote(pr_fields['path'])
        # TODO: figure out what to do with query/fragment... one step at a time
        return ParseResult(**pr_fields)

    @classmethod
    def _pr_to_fields(cls, pr, guessing=False):
        """ParseResult is a tuple so immutable, which complicates adjusting it

        This function converts ParseResult into dict"""
        if pr.params:
            (lgr.debug if guessing else lgr.warning)(
                "ParseResults contains params %s, which will be ignored",
                repr(pr.params),)

        hostname_port = pr.netloc.split('@')[-1]
        is_ipv6 = hostname_port.count(':') >= 2
        # can't use just pr._asdict since we care to ask those properties
        # such as .port , .hostname etc
        # Forcing '' instead of None since those properties (.hostname), .password,
        # .username return None if not available and we decided to uniformize
        if is_ipv6:
            rem = re.match(r'\[(?P<hostname>.*)\]:(?P<port>\d+)', hostname_port)
            if rem:
                hostname, port = rem.groups()
                port = int(port)
            else:
                hostname, port = hostname_port, ''

            def _getattr(pr, f):
                """Helper for custom handling in case of ipv6 addresses which blows
                stock ParseResults logic"""
                if f == 'port':
                    # for now not supported at all, so
                    return port
                elif f == 'hostname':
                    return hostname
                else:
                    return getattr(pr, f)
        else:
            _getattr = getattr

        return {f: (_getattr(pr, f) or '') for f in cls._FIELDS}

    #
    # Access helpers
    #

    def _parse_qs(self, s, auto_delist=True):
        """Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values"""
        if not s:
            return {}
        out = map_items(ensure_unicode, OrderedDict(parse_qsl(s, 1)))
        if not auto_delist:
            return out
        for k in out:
            v = out[k]
            if isinstance(v, list) and len(v) == 1:
                v = v[0]
                out[k] = None if v == '' else v
        return out

    @property
    def query_dict(self):
        return self._parse_qs(self.query)

    @property
    def fragment_dict(self):
        return self._parse_qs(self.fragment)

    def _windows_local_path(self,
                            support_unc: bool = False
                            ) -> str:
        """Convert the URL to a local path on windows, supports UNC and git-annex"""

        # RFC1738 and RFC3986 both forbid unescaped backslash characters in
        # URLs, and therefore also in the path-component of file:-URLs. We
        # assume here that any backslash present in a file-URL is a relict of a
        # verbatim copy of a Windows-style path.
        unified_path = self.path.replace('\\', '/')
        local_path = url2pathname(unified_path)

        # We support UNC notation, and the "special" git-annex drive encoding
        # scheme, i.e. netloc is the drive letter plus a colon.
        # NB, this if clause will not evaluate to True, because our caller
        # filters out net locations with
        if self.netloc:
            if re.match('^[a-zA-Z]:$', self.netloc):
                # This is the git-annex case, i.e. drive spec in netloc
                return self.netloc + local_path
            if support_unc:
                return '\\\\' + self.netloc + local_path
            raise ValueError("Unsupported file: URL: {self}")
        return local_path

    @property
    def localpath(self):
        if self.scheme != 'file':
            raise ValueError(
                "Non 'file://' URL cannot be resolved to a local path")

        # If there is a hostname, we might want to convert to UNC, unless the
        # hostname has the form of a 'Windows drive letter' on windows
        hostname = self.hostname
        if not (hostname in (None, '', 'localhost', '::1')
                or hostname.startswith('127.')
                or re.match('^[a-zA-Z]:$', self.netloc)):
            if on_windows:
                return self._windows_local_path(support_unc=self.support_unc)
            raise ValueError("file:// URL does not point to 'localhost'")

        if on_windows:
            return self._windows_local_path(support_unc=True)

        return url2pathname(self.path)


class PathRI(RI):
    """RI pointing to a (local) file/directory"""
    def as_str(self):
        return self.path

    @classmethod
    def _str_to_fields(cls, url_str):
        # str() to be compatible with pathlib objects
        return dict(path=str(url_str))

    @property
    def localpath(self):
        return str(Path(self.path))

    @property
    def posixpath(self):
        if is_windows_path(self.path):
            win_split = win_splitdrive(self.path)
            return "/" + win_split[0][0] + win_split[1].replace('\\', '/')
        else:
            return self.path


class RegexBasedURLMixin(object):
    """Base class for URLs which we could simple parse using regular expressions"""

    _REGEX = None

    # not used ATM but possible ;)
    # @classmethod
    # def is_str_matches(cls, url_str):
    #     return bool(cls._REGEX.match(url_str))

    @classmethod
    def _str_to_fields(cls, url_str):
        re_match = cls._REGEX.match(url_str)
        if not re_match:
            # TODO: custom error?
            raise ValueError(
                "Cannot handle URL '%s': categorized as %r, but does not match syntax.%s"
                % (url_str,
                   cls,
                   " Did you intent to use '///'?" if url_str.startswith('//') else '')
            )
        fields = cls._get_blank_fields()
        fields.update({k: v for k, v in re_match.groupdict().items() if v})
        cls._normalize_fields(fields)
        return fields

    @classmethod
    def _normalize_fields(self, fields):
        """Helper to be ran if any of the fields need to be normalized after parsing"""
        pass


class SSHRI(RI, RegexBasedURLMixin):
    """RI pointing to a remote location reachable via SSH"""

    _FIELDS = RI._FIELDS + (
        'username',
        'hostname',
        'port',
    )

    _REGEX = re.compile(r'((?P<username>\S*)@)?(?P<hostname>[^#/\\:]+)(\:(?P<path>.*))?$')

    @classmethod
    def _normalize_fields(cls, fields):
        if fields['path'] and fields['path'].startswith('//'):
            # Let's normalize for now to avoid multiple leading slashes
            fields['path'] = '/' + fields['path'].lstrip('/')
        # escape path so we have direct representation of the path to work with
        fields['path'] = unescape_ssh_path(fields['path'])

    def as_str(self, escape=False):
        fields = self.fields  # copy so we could escape symbols
        url_fmt = '{hostname}'
        if fields['username']:
            url_fmt = "{username}@" + url_fmt
        if fields['path']:
            url_fmt += ':{path}'
        if escape:
            fields['path'] = escape_ssh_path(fields['path'])
        return url_fmt.format(**fields)

    # TODO:
    # we can "support" localhost:path as localpaths


class DataLadRI(RI, RegexBasedURLMixin):
    """RI pointing to datasets within default DataLad super-dataset"""

    _FIELDS = RI._FIELDS + (
        'remote',
    )

    # For now or forever we don't deal with any fragments or other special stuff
    _REGEX = re.compile(r'//(?P<remote>[^\s/]*)/(?P<path>.*)$')

    # do they need to be normalized??? losing track ...

    def as_str(self):
        return "//{remote}/{path}".format(**self._fields)

    def as_git_url(self):
        """Dereference /// into original URLs which could be used by git for cloning

        Returns
        -------
        str
          URL string to reference the DataLadRI from its /// form
        """
        if self.remote:
            raise NotImplementedError("not supported ATM to reference additional remotes")
        return "{}{}".format(consts.DATASETS_TOPURL, urlquote(self.path))


class GitTransportRI(RI, RegexBasedURLMixin):
    """RI for some other RI with git transport prefix"""

    # TODO: check how crticial to "inherit" RI._FIELDS asking to provide path
    _FIELDS = RI._FIELDS + (
        'transport',
        'RI',
    )

    # Due to poor design, `ri` argument already present in various
    # places intermixed with **kwargs treatment. So we will use RI
    # here instead of ri.
    _REGEX = re.compile(r'(?P<transport>[A-Za-z0-9][A-Za-z0-9+.-]*)::(?P<RI>.*)$')

    def as_str(self):
        return '{self.transport}::{self.RI}'.format(self=self)


def _split_colon(s, maxsplit=1):
    """Split on unescaped colon"""
    return re.compile(r'(?<!\\):').split(s, maxsplit=maxsplit)

# \ should be first to deal with
_SSH_ESCAPED_CHARACTERS = '\\#&;`|*?~<>^()[]{}$\'" '


# TODO: RF using re.sub
def escape_ssh_path(path):
    """Escape all special characters present in the path"""
    for c in _SSH_ESCAPED_CHARACTERS:
        if c in path:
            path = path.replace(c, '\\' + c)
    return path


def unescape_ssh_path(path):
    """Un-escape all special characters present in the path"""
    for c in _SSH_ESCAPED_CHARACTERS[::-1]:
        if c in path:
            path = path.replace('\\' + c, c)
    return path


def parse_url_opts(url):
    """Given a string with url-style query, split into content before # and options as dict"""
    url = URL(url)
    # we need to filter out query and fragment to get the base url
    fields = url.fields
    fields.pop('query')
    fields.pop('fragment')
    opts = url.query_dict
    return str(URL(**fields)), opts


# TODO: should we just define URL.good_for_git or smth like that? ;)
# although git also understands regular paths
def is_url(ri):
    """Returns whether argument is a resource identifier what datalad should treat as a URL

    This includes ssh "urls" which git understands.

    Parameters
    ----------
    ri : str or RI
      The resource identifier (as a string or RI) to "analyze"
    """
    if not isinstance(ri, RI):
        try:
            ri = RI(ri)
        except:  # MIH: MemoryError?
            return False
    return isinstance(ri, (URL, SSHRI))


# TODO: RF to remove duplication
def is_datalad_compat_ri(ri):
    """Returns whether argument is a resource identifier what datalad should treat as a URL

    including its own DataLadRI
    """
    if not isinstance(ri, RI):
        try:
            ri = RI(ri)
        except:  # MIH: MemoryError?
            return False
    return isinstance(ri, (URL, SSHRI, DataLadRI))


# TODO: better name? additionally may be move to SSHRI.is_valid() or sth.
def is_ssh(ri):
    """helper to determine, whether `ri` requires an SSH connection

    Parameters
    ----------
    ri: str or RI

    Returns
    -------
    bool
    """

    # not exactly fitting the doc, but we actually can deal not necessarily with
    # string or RI only, but with everything RI itself can deal with:
    _ri = RI(ri) if not isinstance(ri, RI) else ri

    return isinstance(_ri, SSHRI) \
        or (isinstance(_ri, URL) and _ri.scheme == 'ssh')


def get_local_file_url(fname: str,
                       compatibility: str = 'git-annex',
                       allow_relative_path: bool = True
                       ) -> str:
    """Return OS specific URL pointing to a local file

    Parameters
    ----------
    fname : string
        Filename.  If not absolute, abspath is used
    compatibility : str, optional
        This parameter is only interpreted on Windows systems. If set to
        anything else than 'git', the anchor, e.g. `C:` of `fname` will be put
        into the `file-auth` part, i.e. network location, defined in RFC 8089.
        This option is mainly used to support git-annex specific encoding of
        Windows paths.
    allow_relative_path: bool, optional
        Allow `fname` to be a relative path. The path will be converted to an
        absolute path, by using the current directory as path prefix.
    """
    path_obj = Path(fname)
    if not path_obj.is_absolute():
        if allow_relative_path:
            fname = str(Path(os.getcwd()) / path_obj)
        else:
            raise ValueError('cannot create file-URL with relative path')

    url_path = local_path2url_path(fname, auto_resolve=False)
    if on_windows and compatibility != "git":
        # Work around the way in which git-annex interprets file URLs on
        # Windows. This code path will put the path anchor, e.g. `C:` of `fname`
        # into the network location component of the resulting URL.
        return "file:/" + url_path

    result = "file://" + url_path
    if result != Path(fname).as_uri():
        lgr.warning(
            f"get_local_file_url({fname}, {compatibility}, "
            f"{allow_relative_path}) returned {result} instead of "
            f"{Path(fname).as_uri()}")
    return result


def get_url_cache_filename(url, name=None):
    """Return a filename where to cache online doc from a url"""
    if not name:
        name = "misc"
    cache_dir = opj(cfg.obtain('datalad.locations.cache'), name)
    doc_fname = opj(
        cache_dir,
        '{}-{}.p{}'.format(
            urlsplit(url).netloc,
            md5(url.encode('utf-8')).hexdigest(),
            pickle.HIGHEST_PROTOCOL)
    )
    return doc_fname


def get_cached_url_content(url, name=None, fetcher=None, maxage=None):
    """Loader of a document from a url, which caches loaded instance on disk

    Doesn't do anything smart about http headers etc which could provide
    information for cache/proxy servers for how long to retain etc

    TODO: theoretically it is not network specific at all -- and just a memoize
    pattern, but may be some time we would make it treat headers etc correctly.
    And ATM would support any URL we support via providers/downloaders

    Parameters
    ----------
    fetcher: callable, optional
       Function to call with url if needed to be refetched
    maxage: float, optional
       Age in days to retain valid for.  <0 - would retain forever.  If None -
       would consult the config, 0 - would force to reload
    """
    doc_fname = get_url_cache_filename(url, name)
    if maxage is None:
        maxage = float(cfg.get('datalad.locations.cache-maxage'))

    doc = None
    if os.path.exists(doc_fname) and maxage != 0:

        fage = (time.time() - os.stat(doc_fname).st_mtime)/(24. * 3600)
        if maxage < 0 or fage < maxage:
            try:
                lgr.debug("use cached request result to '%s' from %s", url, doc_fname)
                doc = pickle.load(open(doc_fname, 'rb'))
            except Exception as e:  # it is OK to ignore any error and fall back on the true source
                lgr.warning(
                    "cannot load cache from '%s', fall back to download: %s",
                    doc_fname, CapturedException(e))

    if doc is None:
        if fetcher is None:
            from datalad.downloaders.providers import Providers
            providers = Providers.from_config_files()
            fetcher = providers.fetch

        doc = fetcher(url)
        ensure_dir(dirname(doc_fname))
        # use pickle to store the entire request result dict
        pickle.dump(doc, open(doc_fname, 'wb'))
        lgr.debug("stored result of request to '%s' in %s", url, doc_fname)
    return doc


def download_url(url, dest=None, overwrite=False):
    """Download a file from a URL

    Supports and honors any DataLad "downloader/provider" configuration.

    Parameters
    ----------
    url: str
      Source URL to download from.
    dest: Path-like or None
      Destination file name (file must not exist), or name of a target
      directory (must exists, and filename must be derivable from `url`).
      If None, the downloaded content will be returned as a string.
    overwrite: bool
      Force overwriting an existing destination file.

    Returns
    -------
    str
      Path of the downloaded file, or URL content if `dest` is None.

    Raises
    ------
    DownloadError
      If `dest` already exists and is a file, or if `dest` is a directory
      and no filename could be determined from `url`, or if no file was
      found at the given `url`.
    """
    from datalad.downloaders.providers import Providers
    providers = Providers.from_config_files()
    if dest:
        return providers.download(url, path=str(dest), overwrite=overwrite)
    else:
        return providers.fetch(url)


def local_path2url_path(local_path: str,
                        auto_resolve: bool = False
                        ) -> str:
    """Convert a local path into a URL path component"""
    if not local_path:
        if not auto_resolve:
            raise ValueError("cannot convert empty local path to URL path")
        local_path = os.getcwd()

    url = urlparse(Path(local_path).as_uri())
    if url.netloc:
        raise ValueError(
            f"cannot convert remote path to an URL path: {local_path}")
    return url.path


def url_path2local_path(url_path: str | PurePosixPath,
                        make_absolute: bool = True,
                        ) -> str | Path:

    if isinstance(url_path, PurePosixPath):
        return_path = True
        url_path = str(url_path)
    else:
        return_path = False

    if not url_path or not url_path.startswith("/"):
        # We expect a 'path-absolute' as defined in RFC 3986, therefore the
        # path must begin with a slash.
        raise ValueError(
            f"url path does not start with '/': {url_path}, and is therefore "
            f"not an absolute-path as defined in RFC 8089")

    if url_path.startswith("//"):
        # We expect a 'path-absolute' as defined in RFC 3986, therefore the
        # first segment must not be empty, i.e. the path must not start with
        # two or more slashes.
        raise ValueError(
            f"url path has empty first segment: {url_path}, and is therefore "
            f"not an absolute-path as defined in RFC 8089")

    return (
        Path(url2pathname(url_path))
        if return_path
        else url2pathname(url_path)
    )


def quote_path(path: str, safe: str = "/") -> str:
    """quote the path component of a URL, takes OS specifics into account"""
    if on_windows:
        if re.match("^/[a-zA-Z]:/", path):
            return path[:3] + quote(path[3:], safe=safe)
    return quote(path, safe=safe)


lgr.log(5, "Done importing support.network")

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user