{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport calendar\nimport email.utils\nimport gzip\nimport os\nimport re\nimport shutil\nimport time\n\nfrom collections import OrderedDict\nfrom os.path import abspath, isabs\n\nfrom six import string_types\nfrom six import iteritems\nfrom six.moves.urllib.request import urlopen, Request\nfrom six.moves.urllib.parse import quote as urlquote, unquote as urlunquote\nfrom six.moves.urllib.parse import urljoin, urlparse, urlsplit, urlunsplit, urlunparse, ParseResult\nfrom six.moves.urllib.parse import parse_qsl\nfrom six.moves.urllib.parse import urlencode\nfrom six.moves.urllib.error import URLError\n\nfrom datalad.utils import on_windows\n\nfrom datalad.utils import auto_repr\n# TODO not sure what needs to use `six` here yet\nimport requests\n\nimport logging\nlgr = logging.getLogger('datalad.network')\n\n\ndef get_response_disposition_filename(s):\n    \"\"\"Given a string s as from HTTP Content-Disposition field in the response\n    return possibly present filename if any\n    \"\"\"\n    if not s:\n        return None\n    # If the response has Content-Disposition, try to get filename from it\n    cd = dict(map(\n        lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),\n        s.split(';')))\n    if 'filename' in cd:\n        filename = cd['filename'].strip(\"\\\"'\")\n        return filename\n    return None\n\n\ndef get_url_disposition_filename(url, headers=None):\n    \"\"\"Get filename as possibly provided by the server in Content-Disposition\n    \"\"\"\n    if headers is None:\n        request = Request(url)\n        r = retry_urlopen(request)\n        # things are different in requests\n        if 'requests.' in str(r.__class__):\n            headers = r.headers\n        else:\n            headers = r.info()\n    else:\n        r = None\n    try:\n        return get_response_disposition_filename(headers.get('Content-Disposition', ''))\n    finally:\n        if r:\n            r.close()\n\n\ndef get_url_straight_filename(url, strip=[], allowdir=False):\n    \"\"\"Get file/dir name of the last path component of the URL\n\n    Parameters\n    ----------\n    strip: list, optional\n      If provided, listed names will not be considered and their\n      parent directory will be selected\n    allowdir: bool, optional\n      If url points to a \"directory\" (ends with /), empty string\n      would be returned unless allowdir is True, in which case the\n      name of the directory would be returned\n    \"\"\"\n    path = urlunquote(urlsplit(url).path)\n    path_parts = path.split('/')\n\n    if allowdir:\n        # strip empty ones\n        while len(path_parts) > 1 and not path_parts[-1]:\n            path_parts = path_parts[:-1]\n\n    if strip:\n        while path_parts and path_parts[-1] in strip:\n            path_parts = path_parts[:-1]\n\n    if path_parts:\n        return path_parts[-1]\n    else:\n        return None\n\n\ndef get_url_filename(url, headers=None, strip=[]):\n    \"\"\"Get filename from the url, first consulting server about Content-Disposition\n    \"\"\"\n    filename = get_url_disposition_filename(url, headers)\n    if filename:\n        return filename\n    return get_url_straight_filename(url, strip=[])\n\n\ndef get_url_response_stamp(url, response_info):\n    size, mtime = None, None\n    if 'Content-length' in response_info:\n        size = int(response_info['Content-length'])\n    if 'Last-modified' in response_info:\n        mtime = calendar.timegm(email.utils.parsedate(\n            response_info['Last-modified']))\n    return dict(size=size, mtime=mtime, url=url)\n\n\ndef get_tld(url):\n    \"\"\"Return top level domain from a url\n\n    Parameters\n    ----------\n    url : str\n    \"\"\"\n    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld\n    if not url.strip():\n        raise ValueError(\"Empty URL has no TLD\")\n    rec = urlsplit(url)\n    if not rec.netloc:\n        if not rec.scheme:\n            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'\n            return url.split('/', 1)[0]\n        else:\n            raise ValueError(\"It seems that only the scheme was provided without the net location/TLD\")\n    return rec.netloc\n\n\nfrom email.utils import parsedate_tz, mktime_tz\ndef rfc2822_to_epoch(datestr):\n    \"\"\"Given rfc2822 date/time format, return seconds since epoch\"\"\"\n    return mktime_tz(parsedate_tz(datestr))\n\n\nimport calendar\nfrom datetime import datetime\ndef iso8601_to_epoch(datestr):\n    return calendar.timegm(datetime.strptime(datestr, \"%Y-%m-%dT%H:%M:%S.%fZ\").timetuple())\n\n\ndef __urlopen_requests(url, header_vals=None):\n    # XXX Workaround for now for ... broken code\n    if isinstance(url, Request):\n        url = url.get_full_url()\n    return requests.Session().get(url)\n\n\ndef retry_urlopen(url, retries=3):\n    for t in range(retries):\n        try:\n            return __urlopen_requests(url)\n        except URLError as e:\n            lgr.warn(\"Received exception while reading %s: %s\" % (url, e))\n            if t == retries - 1:\n                # if we have reached allowed number of retries -- reraise\n                raise\n\n\ndef is_url_quoted(url):\n    \"\"\"Return either URL looks being already quoted\n    \"\"\"\n    try:\n        url_ = urlunquote(url)\n        return url != url_\n    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)\n        return False\n\n\ndef same_website(url_rec, u_rec):\n    \"\"\"Decide either a link leads to external site\n\n    Parameters\n    ----------\n    url_rec: ParseResult\n      record for original url\n    u_rec: ParseResult\n      record for new url\n    \"\"\"\n    if isinstance(url_rec, string_types):\n        url_rec = urlparse(url_rec)\n    if isinstance(u_rec, string_types):\n        u_rec = urlparse(u_rec)\n    return (url_rec.netloc == u_rec.netloc)\n    # todo: collect more of sample cases.\n    # disabled below check while working on ratholeradio, since links\n    # could go to the parent and that is ok.  Figure out when it was\n    # desired not to go to the parent -- we might need explicit option\n    # and u_rec.path.startswith(url_rec.path)):\n\n\ndef dlurljoin(u_path, url):\n    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO\n    if url_rec.scheme:\n        # independent full url, so just return it\n        return url\n    if u_path.endswith('/'):  # should here be also a scheme use?\n        if url.startswith('/'): # jump to the root\n            u_path_rec = urlparse(u_path)\n            return urljoin(urlunparse((u_path_rec.scheme, u_path_rec.netloc, '','','','')), url)\n        else:\n            return os.path.join(u_path, url)\n    # TODO: recall where all this dirname came from and bring into the test\n    return urljoin(os.path.dirname(u_path) + '/', url)\n\n\n# TODO should it be a node maybe?\nclass SimpleURLStamper(object):\n    \"\"\"Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header\n    \"\"\"\n    def __init__(self, mode='full'):\n        self.mode = mode\n\n    def __call__(self, url):\n        # Extracted from above madness\n        # TODO: add mode alike to 'relaxed' where we would not\n        # care about content-disposition filename\n        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way\n        request = Request(url)\n\n        # No traffic compression since we do not know how to identify\n        # exactly either it has to be decompressed\n        # request.add_header('Accept-encoding', 'gzip,deflate')\n        #\n        # TODO: think about stamping etc -- we seems to be redoing\n        # what git-annex does for us already... not really\n        r = retry_urlopen(request)\n        try:\n            r_info = r.info()\n            r_stamp = get_url_response_stamp(url, r_info)\n\n            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)\n        finally:\n            r.close()\n\n\n# TODO:  make it consistent/clear at what stage % encoding/decoding happens!\n# now it is a mix!\n\n#\n# Useful functionality in requests.models\n#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance\n#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...\n#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc\n#\n\n#@auto_repr\n# TODO: Well -- it is more of a URI than URL I guess\nclass URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)\n\n    # def __getattribute__(self, item):\n    #     if item.startswith('_') or item not in URL._FIELDS:\n    #         return super(URL, self).__getattribute__(item)\n    #     else:\n    #         return self._fields[item]\n\n# Bind properties to access fields (without overriding __getattr*)\n# This one doesn't work -- I guess due to absent binding of f value\n# to the context\n#for f in URL._FIELDS:\n#   setattr(URL, f, property(lambda self: self._fields[f]))\n# These work but ugly duplication\n# URL.hostname = property(lambda self: self._fields['hostname'])\n# URL.path = property(lambda self: self._fields['path'])\n# URL.query = property(lambda self: self._fields['query'])\n# URL.scheme = property(lambda self: self._fields['scheme'])\n# URL.fragment = property(lambda self: self._fields['fragment'])\n# URL.username = property(lambda self: self._fields['username'])\n# URL.password = property(lambda self: self._fields['password'])\n# URL.port = property(lambda self: self._fields['port'])\nfor f in URL._FIELDS:\n    exec(\"URL.%s = property(lambda self: self._fields['%s'])\" % (f, f))\n\n\ndef _split_colon(s, maxsplit=1):\n    \"\"\"Split on unescaped colon\"\"\"\n    return re.compile(r'(?<!\\\\):').split(s, maxsplit=maxsplit)\n\n\ndef parse_url_opts(url):\n    \"\"\"Given a string with url-style query, split into content before # and options as dict\"\"\"\n    url = URL(url)\n    # we need to filter out query and fragment to get the base url\n    fields = url.fields\n    fields.pop('query')\n    fields.pop('fragment')\n    opts = url.query_dict\n    return str(URL(**fields)), opts\n\n\n# TODO: should we just define URL.good_for_git or smth like that? ;)\n# although git also understands regular paths\ndef is_url(s):\n    \"\"\"Returns whether a string looks like something what datalad should treat as a URL\n\n    This includes ssh \"urls\" which git understands.\n    \"\"\"\n    try:\n        url = URL(s)\n    except:\n        return False\n    implicit = url.is_implicit\n    scheme = url.scheme\n    return scheme in {'ssh:implicit'} or (not implicit and bool(url))\n\n\n#### windows workaround ###\n# TODO: There should be a better way\ndef get_local_file_url(fname):\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    \"\"\"\n    fname = fname if isabs(fname) else abspath(fname)\n    if on_windows:\n        fname_rep = fname.replace('\\\\', '/')\n        furl = \"file:///%s\" % urlquote(fname_rep)\n        lgr.debug(\"Replaced '\\\\' in file\\'s url: %s\" % furl)\n    else:\n        # TODO:  need to fix for all the encoding etc\n        furl = str(URL(scheme='file', path=urlquote(fname)))\n    return furl\n\n\ndef get_local_path_from_url(url):\n    \"\"\"If given a file:// URL, returns a local path, if possible.\n\n    Raises `ValueError` if not possible, for example, if the URL\n    scheme is different, or if the `host` isn't empty or 'localhost'\n\n    The returned path is always absolute.\n    \"\"\"\n    urlparts = urlsplit(url)\n    if not urlparts.scheme == 'file':\n        raise ValueError(\n            \"Non 'file://' URL cannot be resolved to a local path\")\n    if not (urlparts.netloc in ('', 'localhost', '::1') \\\n            or urlparts.netloc.startswith('127.')):\n        raise ValueError(\"file:// URL does not point to 'localhost'\")\n    return urlunquote(urlparts.path)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport calendar\nimport email.utils\nimport gzip\nimport os\nimport re\nimport shutil\nimport time\n\nfrom collections import OrderedDict\nfrom os.path import abspath, isabs\n\nfrom six import string_types\nfrom six import iteritems\nfrom six.moves.urllib.request import urlopen, Request\nfrom six.moves.urllib.parse import quote as urlquote, unquote as urlunquote\nfrom six.moves.urllib.parse import urljoin, urlparse, urlsplit, urlunsplit, urlunparse, ParseResult\nfrom six.moves.urllib.parse import parse_qsl\nfrom six.moves.urllib.parse import urlencode\nfrom six.moves.urllib.error import URLError\n\nfrom datalad.utils import on_windows\n\nfrom datalad.utils import auto_repr\n# TODO not sure what needs to use `six` here yet\nimport requests\n\nimport logging\nlgr = logging.getLogger('datalad.network')\n\n\ndef get_response_disposition_filename(s):\n    \"\"\"Given a string s as from HTTP Content-Disposition field in the response\n    return possibly present filename if any\n    \"\"\"\n    if not s:\n        return None\n    # If the response has Content-Disposition, try to get filename from it\n    cd = dict(map(\n        lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),\n        s.split(';')))\n    if 'filename' in cd:\n        filename = cd['filename'].strip(\"\\\"'\")\n        return filename\n    return None\n\n\ndef get_url_disposition_filename(url, headers=None):\n    \"\"\"Get filename as possibly provided by the server in Content-Disposition\n    \"\"\"\n    if headers is None:\n        request = Request(url)\n        r = retry_urlopen(request)\n        # things are different in requests\n        if 'requests.' in str(r.__class__):\n            headers = r.headers\n        else:\n            headers = r.info()\n    else:\n        r = None\n    try:\n        return get_response_disposition_filename(headers.get('Content-Disposition', ''))\n    finally:\n        if r:\n            r.close()\n\n\ndef get_url_straight_filename(url, strip=[], allowdir=False):\n    \"\"\"Get file/dir name of the last path component of the URL\n\n    Parameters\n    ----------\n    strip: list, optional\n      If provided, listed names will not be considered and their\n      parent directory will be selected\n    allowdir: bool, optional\n      If url points to a \"directory\" (ends with /), empty string\n      would be returned unless allowdir is True, in which case the\n      name of the directory would be returned\n    \"\"\"\n    path = urlunquote(urlsplit(url).path)\n    path_parts = path.split('/')\n\n    if allowdir:\n        # strip empty ones\n        while len(path_parts) > 1 and not path_parts[-1]:\n            path_parts = path_parts[:-1]\n\n    if strip:\n        while path_parts and path_parts[-1] in strip:\n            path_parts = path_parts[:-1]\n\n    if path_parts:\n        return path_parts[-1]\n    else:\n        return None\n\n\ndef get_url_filename(url, headers=None, strip=[]):\n    \"\"\"Get filename from the url, first consulting server about Content-Disposition\n    \"\"\"\n    filename = get_url_disposition_filename(url, headers)\n    if filename:\n        return filename\n    return get_url_straight_filename(url, strip=[])\n\n\ndef get_url_response_stamp(url, response_info):\n    size, mtime = None, None\n    if 'Content-length' in response_info:\n        size = int(response_info['Content-length'])\n    if 'Last-modified' in response_info:\n        mtime = calendar.timegm(email.utils.parsedate(\n            response_info['Last-modified']))\n    return dict(size=size, mtime=mtime, url=url)\n\n\ndef get_tld(url):\n    \"\"\"Return top level domain from a url\n\n    Parameters\n    ----------\n    url : str\n    \"\"\"\n    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld\n    if not url.strip():\n        raise ValueError(\"Empty URL has no TLD\")\n    rec = urlsplit(url)\n    if not rec.netloc:\n        if not rec.scheme:\n            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'\n            return url.split('/', 1)[0]\n        else:\n            raise ValueError(\"It seems that only the scheme was provided without the net location/TLD\")\n    return rec.netloc\n\n\nfrom email.utils import parsedate_tz, mktime_tz\ndef rfc2822_to_epoch(datestr):\n    \"\"\"Given rfc2822 date/time format, return seconds since epoch\"\"\"\n    return mktime_tz(parsedate_tz(datestr))\n\n\nimport calendar\nfrom datetime import datetime\ndef iso8601_to_epoch(datestr):\n    return calendar.timegm(datetime.strptime(datestr, \"%Y-%m-%dT%H:%M:%S.%fZ\").timetuple())\n\n\ndef __urlopen_requests(url, header_vals=None):\n    # XXX Workaround for now for ... broken code\n    if isinstance(url, Request):\n        url = url.get_full_url()\n    return requests.Session().get(url)\n\n\ndef retry_urlopen(url, retries=3):\n    for t in range(retries):\n        try:\n            return __urlopen_requests(url)\n        except URLError as e:\n            lgr.warn(\"Received exception while reading %s: %s\" % (url, e))\n            if t == retries - 1:\n                # if we have reached allowed number of retries -- reraise\n                raise\n\n\ndef is_url_quoted(url):\n    \"\"\"Return either URL looks being already quoted\n    \"\"\"\n    try:\n        url_ = urlunquote(url)\n        return url != url_\n    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)\n        return False\n\n\ndef same_website(url_rec, u_rec):\n    \"\"\"Decide either a link leads to external site\n\n    Parameters\n    ----------\n    url_rec: ParseResult\n      record for original url\n    u_rec: ParseResult\n      record for new url\n    \"\"\"\n    if isinstance(url_rec, string_types):\n        url_rec = urlparse(url_rec)\n    if isinstance(u_rec, string_types):\n        u_rec = urlparse(u_rec)\n    return (url_rec.netloc == u_rec.netloc)\n    # todo: collect more of sample cases.\n    # disabled below check while working on ratholeradio, since links\n    # could go to the parent and that is ok.  Figure out when it was\n    # desired not to go to the parent -- we might need explicit option\n    # and u_rec.path.startswith(url_rec.path)):\n\n\ndef dlurljoin(u_path, url):\n    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO\n    if url_rec.scheme:\n        # independent full url, so just return it\n        return url\n    if u_path.endswith('/'):  # should here be also a scheme use?\n        if url.startswith('/'): # jump to the root\n            u_path_rec = urlparse(u_path)\n            return urljoin(urlunparse((u_path_rec.scheme, u_path_rec.netloc, '','','','')), url)\n        else:\n            return os.path.join(u_path, url)\n    # TODO: recall where all this dirname came from and bring into the test\n    return urljoin(os.path.dirname(u_path) + '/', url)\n\n\n# TODO should it be a node maybe?\nclass SimpleURLStamper(object):\n    \"\"\"Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header\n    \"\"\"\n    def __init__(self, mode='full'):\n        self.mode = mode\n\n    def __call__(self, url):\n        # Extracted from above madness\n        # TODO: add mode alike to 'relaxed' where we would not\n        # care about content-disposition filename\n        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way\n        request = Request(url)\n\n        # No traffic compression since we do not know how to identify\n        # exactly either it has to be decompressed\n        # request.add_header('Accept-encoding', 'gzip,deflate')\n        #\n        # TODO: think about stamping etc -- we seems to be redoing\n        # what git-annex does for us already... not really\n        r = retry_urlopen(request)\n        try:\n            r_info = r.info()\n            r_stamp = get_url_response_stamp(url, r_info)\n\n            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)\n        finally:\n            r.close()\n\n\n# TODO:  make it consistent/clear at what stage % encoding/decoding happens!\n# now it is a mix!\n\n#\n# Useful functionality in requests.models\n#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance\n#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...\n#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc\n#\n\n#@auto_repr\n# TODO: Well -- it is more of a URI than URL I guess\nclass URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading / after the hostname\n        # Will preserve hostname as is, since we are trying to be as silly as ssh\n        # where it tries to resolve hostnames with /\n        lhostname = len(self.hostname)\n\n        url = url[:lhostname] + \\\n              url[lhostname:].replace(\n                  '/',\n                  ':/' if self.path.startswith('/') else ':',\n                  1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            parts = _split_colon(url)\n            if fields['path'] and '@' in fields['path'] or len(parts) > 1:\n                # user@host:path/sp1\n                # or host_name: (hence parts check)\n                # TODO: we need a regex to catch those really, parts check is not suff\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)\n\n    # def __getattribute__(self, item):\n    #     if item.startswith('_') or item not in URL._FIELDS:\n    #         return super(URL, self).__getattribute__(item)\n    #     else:\n    #         return self._fields[item]\n\n# Bind properties to access fields (without overriding __getattr*)\n# This one doesn't work -- I guess due to absent binding of f value\n# to the context\n#for f in URL._FIELDS:\n#   setattr(URL, f, property(lambda self: self._fields[f]))\n# These work but ugly duplication\n# URL.hostname = property(lambda self: self._fields['hostname'])\n# URL.path = property(lambda self: self._fields['path'])\n# URL.query = property(lambda self: self._fields['query'])\n# URL.scheme = property(lambda self: self._fields['scheme'])\n# URL.fragment = property(lambda self: self._fields['fragment'])\n# URL.username = property(lambda self: self._fields['username'])\n# URL.password = property(lambda self: self._fields['password'])\n# URL.port = property(lambda self: self._fields['port'])\nfor f in URL._FIELDS:\n    exec(\"URL.%s = property(lambda self: self._fields['%s'])\" % (f, f))\n\n\ndef _split_colon(s, maxsplit=1):\n    \"\"\"Split on unescaped colon\"\"\"\n    return re.compile(r'(?<!\\\\):').split(s, maxsplit=maxsplit)\n\n\ndef parse_url_opts(url):\n    \"\"\"Given a string with url-style query, split into content before # and options as dict\"\"\"\n    url = URL(url)\n    # we need to filter out query and fragment to get the base url\n    fields = url.fields\n    fields.pop('query')\n    fields.pop('fragment')\n    opts = url.query_dict\n    return str(URL(**fields)), opts\n\n\n# TODO: should we just define URL.good_for_git or smth like that? ;)\n# although git also understands regular paths\ndef is_url(s):\n    \"\"\"Returns whether a string looks like something what datalad should treat as a URL\n\n    This includes ssh \"urls\" which git understands.\n    \"\"\"\n    try:\n        url = URL(s)\n    except:\n        return False\n    implicit = url.is_implicit\n    scheme = url.scheme\n    return scheme in {'ssh:implicit'} or (not implicit and bool(url))\n\n\n#### windows workaround ###\n# TODO: There should be a better way\ndef get_local_file_url(fname):\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    \"\"\"\n    fname = fname if isabs(fname) else abspath(fname)\n    if on_windows:\n        fname_rep = fname.replace('\\\\', '/')\n        furl = \"file:///%s\" % urlquote(fname_rep)\n        lgr.debug(\"Replaced '\\\\' in file\\'s url: %s\" % furl)\n    else:\n        # TODO:  need to fix for all the encoding etc\n        furl = str(URL(scheme='file', path=urlquote(fname)))\n    return furl\n\n\ndef get_local_path_from_url(url):\n    \"\"\"If given a file:// URL, returns a local path, if possible.\n\n    Raises `ValueError` if not possible, for example, if the URL\n    scheme is different, or if the `host` isn't empty or 'localhost'\n\n    The returned path is always absolute.\n    \"\"\"\n    urlparts = urlsplit(url)\n    if not urlparts.scheme == 'file':\n        raise ValueError(\n            \"Non 'file://' URL cannot be resolved to a local path\")\n    if not (urlparts.netloc in ('', 'localhost', '::1') \\\n            or urlparts.netloc.startswith('127.')):\n        raise ValueError(\"file:// URL does not point to 'localhost'\")\n    return urlunquote(urlparts.path)\n",
            "header": {
                "file": {
                    "path": "datalad/support/network.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "88fbba7e0",
                "new_commit_hash": "c5510d6c9",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/network.py"
            },
            "plus_file": {
                "path": "b/datalad/support/network.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 348,
                            "length": 10
                        },
                        "new_range": {
                            "start": 348,
                            "length": 16
                        },
                        "section": "class URL(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        pref = 'ssh:implicit://'",
                                "type": "context"
                            },
                            {
                                "content": "        assert(url.startswith(pref))",
                                "type": "context"
                            },
                            {
                                "content": "        url = url[len(pref):]",
                                "type": "context"
                            },
                            {
                                "content": "        # and we should replace leading /",
                                "type": "deleted"
                            },
                            {
                                "content": "        url = url.replace('/',",
                                "type": "deleted"
                            },
                            {
                                "content": "                          ':/' if self.path.startswith('/') else ':',",
                                "type": "deleted"
                            },
                            {
                                "content": "                          1)",
                                "type": "deleted"
                            },
                            {
                                "content": "        # and we should replace leading / after the hostname",
                                "type": "added"
                            },
                            {
                                "content": "        # Will preserve hostname as is, since we are trying to be as silly as ssh",
                                "type": "added"
                            },
                            {
                                "content": "        # where it tries to resolve hostnames with /",
                                "type": "added"
                            },
                            {
                                "content": "        lhostname = len(self.hostname)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        url = url[:lhostname] + \\",
                                "type": "added"
                            },
                            {
                                "content": "              url[lhostname:].replace(",
                                "type": "added"
                            },
                            {
                                "content": "                  '/',",
                                "type": "added"
                            },
                            {
                                "content": "                  ':/' if self.path.startswith('/') else ':',",
                                "type": "added"
                            },
                            {
                                "content": "                  1)",
                                "type": "added"
                            },
                            {
                                "content": "        return url",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def __str_datalad__(self):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "class",
                            "name": "URL",
                            "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 265,
                            "end_lineno": 569,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "method",
                            "name": "URL.__str_ssh__",
                            "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 345,
                            "end_lineno": 355,
                            "parent": {
                                "file_name": "datalad/support/network.py",
                                "type": "class",
                                "name": "URL",
                                "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 265,
                                "end_lineno": 569,
                                "parent": null
                            }
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 477,
                            "length": 7
                        },
                        "new_range": {
                            "start": 483,
                            "length": 6
                        },
                        "section": "class URL(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    def _set_from_str(self, url):",
                                "type": "context"
                            },
                            {
                                "content": "        fields = self._pr_to_fields(urlparse(url))",
                                "type": "context"
                            },
                            {
                                "content": "        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "        # Special treatments",
                                "type": "context"
                            },
                            {
                                "content": "        # file:///path should stay file:",
                                "type": "context"
                            },
                            {
                                "content": "        if fields['scheme'] and fields['scheme'] not in {'file'} \\",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "method",
                            "name": "URL._set_from_str",
                            "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 477,
                            "end_lineno": 531,
                            "parent": {
                                "file_name": "datalad/support/network.py",
                                "type": "class",
                                "name": "URL",
                                "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 265,
                                "end_lineno": 569,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "class",
                            "name": "URL",
                            "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 265,
                            "end_lineno": 569,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 489,
                            "length": 8
                        },
                        "new_range": {
                            "start": 494,
                            "length": 11
                        },
                        "section": "class URL(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        if not fields['scheme'] and not fields['hostname']:",
                                "type": "context"
                            },
                            {
                                "content": "            if fields['path'] and '@' in fields['path']:",
                                "type": "deleted"
                            },
                            {
                                "content": "            parts = _split_colon(url)",
                                "type": "added"
                            },
                            {
                                "content": "            if fields['path'] and '@' in fields['path'] or len(parts) > 1:",
                                "type": "added"
                            },
                            {
                                "content": "                # user@host:path/sp1",
                                "type": "context"
                            },
                            {
                                "content": "                # or host_name: (hence parts check)",
                                "type": "added"
                            },
                            {
                                "content": "                # TODO: we need a regex to catch those really, parts check is not suff",
                                "type": "added"
                            },
                            {
                                "content": "                fields['scheme'] = 'ssh:implicit'",
                                "type": "context"
                            },
                            {
                                "content": "            elif url.startswith('//'):",
                                "type": "context"
                            },
                            {
                                "content": "                # e.g. // or ///path",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "method",
                            "name": "URL._set_from_str",
                            "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 477,
                            "end_lineno": 531,
                            "parent": {
                                "file_name": "datalad/support/network.py",
                                "type": "class",
                                "name": "URL",
                                "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 265,
                                "end_lineno": 569,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/support/network.py",
                            "type": "class",
                            "name": "URL",
                            "content": "class URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 265,
                            "end_lineno": 569,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport logging\n\nfrom collections import OrderedDict\n\nfrom .utils import eq_, neq_, ok_, nok_, assert_raises\nfrom .utils import skip_if_on_windows\nfrom .utils import swallow_logs\nfrom .utils import assert_re_in\nfrom .utils import get_most_obscure_supported_name\n\nfrom ..support.network import same_website, dlurljoin\nfrom ..support.network import get_tld\nfrom ..support.network import get_url_straight_filename\nfrom ..support.network import get_response_disposition_filename\nfrom ..support.network import parse_url_opts\nfrom ..support.network import URL\nfrom ..support.network import _split_colon\nfrom ..support.network import is_url\nfrom ..support.network import get_local_file_url\nfrom ..support.network import get_local_path_from_url\n\ndef test_same_website():\n    ok_(same_website(\"http://a.b\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"https://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"https://a.b/2014/01/xxx/\"))\n\n\ndef test_get_tld():\n    eq_(get_tld('http://example.com'), 'example.com')\n    eq_(get_tld('http://example.com/1'), 'example.com')\n    eq_(get_tld('http://example.com/1/2'), 'example.com')\n    eq_(get_tld('example.com/1/2'), 'example.com')\n    eq_(get_tld('s3://example.com/1/2'), 'example.com')\n    assert_raises(ValueError, get_tld, \"\")\n    assert_raises(ValueError, get_tld, \"s3://\")\n    assert_raises(ValueError, get_tld, \"http://\")\n\ndef test_dlurljoin():\n    eq_(dlurljoin('http://a.b/', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/page', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/dir/', 'f'), 'http://a.b/dir/f')\n    eq_(dlurljoin('http://a.b/dir/', 'http://url'), 'http://url')\n    eq_(dlurljoin('http://a.b/dir/', '/'), 'http://a.b/')\n    eq_(dlurljoin('http://a.b/dir/', '/x/y'), 'http://a.b/x/y')\n\ndef _test_get_url_straight_filename(suf):\n    eq_(get_url_straight_filename('http://a.b/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')\n\ndef test_get_url_straight_filename():\n    yield _test_get_url_straight_filename, ''\n    yield _test_get_url_straight_filename, '#'\n    yield _test_get_url_straight_filename, '#tag'\n    yield _test_get_url_straight_filename, '#tag/obscure'\n    yield _test_get_url_straight_filename, '?param=1'\n    yield _test_get_url_straight_filename, '?param=1&another=/'\n\nfrom ..support.network import rfc2822_to_epoch\ndef test_rfc2822_to_epoch():\n    eq_(rfc2822_to_epoch(\"Thu, 16 Oct 2014 01:16:17 EDT\"), 1413436577)\n\n\ndef test_get_response_disposition_filename():\n    eq_(get_response_disposition_filename('attachment;filename=\"Part1-Subjects1-99.tar\"'), \"Part1-Subjects1-99.tar\")\n    eq_(get_response_disposition_filename('attachment'), None)\n\n\ndef test_parse_url_opts():\n    url = 'http://map.org/api/download/?id=157'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '157'}))\n\n    url = 's3://bucket/save/?key=891'\n    output = parse_url_opts(url)\n    eq_(output, ('s3://bucket/save/', {'key': '891'}))\n\n    url = 'http://map.org/api/download/?id=98&code=13'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '98', 'code': '13'}))\n\n\ndef test_split_colon():\n    eq_(_split_colon('a:b'), ['a', 'b'])\n    eq_(_split_colon('a:b:c'), ['a', 'b:c'])\n    eq_(_split_colon('a:b:c', 2), ['a', 'b', 'c'])\n    eq_(_split_colon('ab'), ['ab'])\n    eq_(_split_colon(r'a\\:b'), [r'a\\:b'])\n\n\ndef test_url_eq():\n    eq_(URL(), URL())\n    neq_(URL(), URL(hostname='x'))\n\n\ndef _check_url(url, **fields):\n    \"\"\"just a helper to carry out few checks on urls\"\"\"\n    url_ = URL(**fields)\n    eq_(URL(url), url_)\n    eq_(url, url_)  # just in case ;)  above should fail first if smth is wrong\n    eq_(url, str(url_))  # that we can reconstruct it EXACTLY on our examples\n    # and that we have access to all those fields\n    nok_(set(fields).difference(URL._FIELDS))\n    for f, v in fields.items():\n        eq_(getattr(url_, f), v)\n\n\ndef test_url_base():\n    # Basic checks\n    assert_raises(ValueError, URL, \"http://example.com\", hostname=\"example.com\")\n\n    url = URL(\"http://example.com\")\n    eq_(url.hostname, 'example.com')\n    eq_(url.scheme, 'http')\n    eq_(url.port, '')  # not specified -- empty strings\n    eq_(url.username, '')  # not specified -- empty strings\n    nok_(url.is_implicit)\n    eq_(repr(url), \"URL(hostname='example.com', scheme='http')\")\n    eq_(url, \"http://example.com\")  # automagic coercion in __eq__\n\n    neq_(URL(), URL(hostname='x'))\n\n    smth = URL('smth')\n    eq_(smth.hostname, '')\n    ok_(bool(smth))\n    nok_(bool(URL()))\n\n\ndef test_url_samples():\n    _check_url(\"http://example.com\", scheme='http', hostname=\"example.com\")\n    # \"complete\" one for classical http\n    _check_url(\"http://user:pw@example.com/p/sp?p1=v1&p2=v2#frag\",\n        scheme='http', hostname=\"example.com\", username='user', password='pw', path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n\n    # sample one for ssh with specifying the scheme\n    # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n    _check_url(\"ssh://host/path/sp1\", scheme='ssh', hostname='host', path='/path/sp1')\n    _check_url(\"user@host:path/sp1\",\n               scheme='ssh:implicit', hostname='host', path='path/sp1', username='user')\n    _check_url(\"host:path/sp1\", scheme='ssh:implicit', hostname='host', path='path/sp1')\n    _check_url(\"host:path\", scheme='ssh:implicit', hostname='host', path='path')\n    _check_url(\"host:/path\", scheme='ssh:implicit', hostname='host', path='/path')\n    # TODO!!!  should this be a legit URL like this?\n    # _check_url(\"host\", scheme='ssh:implicit', hostname='host'))\n    eq_(repr(URL(\"host:path\")), \"URL(hostname='host', path='path', scheme='ssh:implicit')\")\n\n    # And now perspective 'datalad:implicit' urls pointing to the canonical center location\n    _check_url(\"///\", scheme='datalad:implicit', path='/')\n    _check_url(\"///p/s1\", scheme='datalad:implicit', path='/p/s1')\n    # could be considered by someone as \"URI reference\" relative to scheme\n    _check_url(\"//a/\", scheme='datalad:implicit', path='/', hostname='a')\n    _check_url(\"//a/data\", scheme='datalad:implicit', path='/data', hostname='a')\n\n    # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n    # or not so custom as\n    _check_url(\"hg+https://host/user/proj\",\n        scheme=\"hg+https\", hostname='host', path='/user/proj')\n    # \"old\" style\n    _check_url(\"dl+archive:KEY/path/sp1#size=123\",\n        scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n    # \"new\" style\n    _check_url(\"dl+archive:KEY#path=path/sp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n    # actually above one is probably wrong since we need to encode the path\n    _check_url(\"dl+archive:KEY#path=path%2Fbsp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n\n    #https://en.wikipedia.org/wiki/File_URI_scheme\n    _check_url(\"file://host\", scheme='file', hostname='host')\n    _check_url(\"file://host/path/sp1\", scheme='file', hostname='host', path='/path/sp1')\n    _check_url(\"file:///path/sp1\", scheme='file', path='/path/sp1')\n    _check_url(\"file:///~/path/sp1\", scheme='file', path='/~/path/sp1')\n    # not sure but let's check\n    _check_url(\"file:///c:/path/sp1\", scheme='file', path='/c:/path/sp1')\n\n    # and now implicit paths or actually they are also \"URI references\"\n    _check_url(\"f\", scheme='file:implicit', path='f')\n    _check_url(\"f/s1\", scheme='file:implicit', path='f/s1')\n    _check_url(\"/f\", scheme='file:implicit', path='/f')\n    _check_url(\"/f/s1\", scheme='file:implicit', path='/f/s1')\n\n    # some github ones, just to make sure\n    _check_url(\"git://host/user/proj\", scheme=\"git\", hostname=\"host\", path=\"/user/proj\")\n    _check_url(\"git@host:user/proj\",\n               scheme=\"ssh:implicit\", hostname=\"host\", path=\"user/proj\", username='git')\n\n    _check_url('weired:/', scheme='ssh:implicit', hostname='weired', path='/')\n    # FAIL?!!!  since schema is not allowing some symbols so we need to add additional\n    # check\n    #_check_url('weired_url:/', scheme='ssh:implicit', hostname='weired_url', path='/')\n\n    # check that we are getting a warning logged when url can't be reconstructed\n    # precisely\n    # actually failed to come up with one -- becomes late here\n    #_check_url(\"http://host///..//p\", scheme='http', path='/..//p')\n\n    # actually this one is good enough to trigger a warning and I still don't know\n    # what it should exactly be!?\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        weired_str = 'weired://'\n        weired_url = URL(weired_str)\n        repr(weired_url)\n        assert_re_in('Parsed version of url .weired. differs from original .weired://.',\n                     cml.out)\n        # but we store original str\n        eq_(str(weired_url), weired_str)\n        neq_(weired_url._as_str(), weired_str)\n\n\ndef test_url_compose_archive_one():\n    url = URL(scheme='dl+archive', path='KEY',\n              fragment=OrderedDict((('path', 'f/p/ s+'), ('size', 30))))\n    # funny - space is encoded as + but + is %2B\n    eq_(str(url), 'dl+archive:KEY#path=f/p/+s%2B&size=30')\n    eq_(url.fragment_dict, {'path': 'f/p/ s+', 'size': '30'})\n\n\ndef test_url_fragments_and_query():\n    url = URL(hostname=\"host\", query=OrderedDict((('a', 'x/b'), ('b', 'y'))))\n    eq_(str(url), '//host?a=x%2Fb&b=y')\n    eq_(url.query, 'a=x%2Fb&b=y')\n    eq_(url.query_dict, {'a': 'x/b', 'b': 'y'})\n\n    url = URL(hostname=\"host\", fragment=OrderedDict((('b', 'x/b'), ('a', 'y'))))\n    eq_(str(url), '//host#b=x/b&a=y')\n    eq_(url.fragment, 'b=x/b&a=y')\n    eq_(url.fragment_dict, {'a': 'y', 'b': 'x/b'})\n\n    fname = get_most_obscure_supported_name()\n    url = URL(hostname=\"host\", fragment={'a': fname})\n    eq_(url.fragment_dict, {'a': fname})\n\n\ndef test_url_dicts():\n    eq_(URL(\"http://host\").query_dict, {})\n\n\n@skip_if_on_windows\ndef test_get_url_path_on_fileurls():\n    eq_(URL('file:///a').path, '/a')\n    eq_(URL('file:///a/b').path, '/a/b')\n    eq_(URL('file:///a/b#id').path, '/a/b')\n    eq_(URL('file:///a/b?whatever').path, '/a/b')\n\n\ndef test_is_url():\n    ok_(is_url('file://localhost/some'))\n    ok_(is_url('http://localhost'))\n    ok_(is_url('ssh://me@localhost'))\n    # in current understanding it is indeed a url but an 'ssh:implicit', not just\n    # a useless scheme=weird with a hope to point to a netloc\n    with swallow_logs():\n        ok_(is_url('weired://'))\n    nok_(is_url('relative'))\n    nok_(is_url('/absolute'))\n    nok_(is_url('like@sshlogin'))\n    nok_(is_url(''))\n    nok_(is_url(' '))\n\n\ndef test_get_local_file_url_linux():\n    eq_(get_local_file_url('/a'), 'file:///a')\n    eq_(get_local_file_url('/a/b/c'), 'file:///a/b/c')\n    eq_(get_local_file_url('/a~'), 'file:///a%7E')\n    eq_(get_local_file_url('/a b/'), 'file:///a%20b/')\n\n\ndef test_get_local_path_from_url():\n    assert_raises(ValueError, get_local_path_from_url, 'http://some')\n    assert_raises(ValueError, get_local_path_from_url, 'file://elsewhere/some')\n    # invalid URL -- is it?  just that 'hostname' is some and no path\n    assert_raises(ValueError, get_local_path_from_url, 'file://some')\n    eq_(get_local_path_from_url('file:///some'), '/some')\n    eq_(get_local_path_from_url('file://localhost/some'), '/some')\n    eq_(get_local_path_from_url('file://::1/some'), '/some')\n    eq_(get_local_path_from_url('file://127.3.4.155/some'), '/some')\n\n\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport logging\n\nfrom collections import OrderedDict\n\nfrom .utils import eq_, neq_, ok_, nok_, assert_raises\nfrom .utils import skip_if_on_windows\nfrom .utils import swallow_logs\nfrom .utils import assert_re_in\nfrom .utils import get_most_obscure_supported_name\n\nfrom ..support.network import same_website, dlurljoin\nfrom ..support.network import get_tld\nfrom ..support.network import get_url_straight_filename\nfrom ..support.network import get_response_disposition_filename\nfrom ..support.network import parse_url_opts\nfrom ..support.network import URL\nfrom ..support.network import _split_colon\nfrom ..support.network import is_url\nfrom ..support.network import get_local_file_url\nfrom ..support.network import get_local_path_from_url\n\ndef test_same_website():\n    ok_(same_website(\"http://a.b\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"https://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"https://a.b/2014/01/xxx/\"))\n\n\ndef test_get_tld():\n    eq_(get_tld('http://example.com'), 'example.com')\n    eq_(get_tld('http://example.com/1'), 'example.com')\n    eq_(get_tld('http://example.com/1/2'), 'example.com')\n    eq_(get_tld('example.com/1/2'), 'example.com')\n    eq_(get_tld('s3://example.com/1/2'), 'example.com')\n    assert_raises(ValueError, get_tld, \"\")\n    assert_raises(ValueError, get_tld, \"s3://\")\n    assert_raises(ValueError, get_tld, \"http://\")\n\ndef test_dlurljoin():\n    eq_(dlurljoin('http://a.b/', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/page', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/dir/', 'f'), 'http://a.b/dir/f')\n    eq_(dlurljoin('http://a.b/dir/', 'http://url'), 'http://url')\n    eq_(dlurljoin('http://a.b/dir/', '/'), 'http://a.b/')\n    eq_(dlurljoin('http://a.b/dir/', '/x/y'), 'http://a.b/x/y')\n\ndef _test_get_url_straight_filename(suf):\n    eq_(get_url_straight_filename('http://a.b/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')\n\ndef test_get_url_straight_filename():\n    yield _test_get_url_straight_filename, ''\n    yield _test_get_url_straight_filename, '#'\n    yield _test_get_url_straight_filename, '#tag'\n    yield _test_get_url_straight_filename, '#tag/obscure'\n    yield _test_get_url_straight_filename, '?param=1'\n    yield _test_get_url_straight_filename, '?param=1&another=/'\n\nfrom ..support.network import rfc2822_to_epoch\ndef test_rfc2822_to_epoch():\n    eq_(rfc2822_to_epoch(\"Thu, 16 Oct 2014 01:16:17 EDT\"), 1413436577)\n\n\ndef test_get_response_disposition_filename():\n    eq_(get_response_disposition_filename('attachment;filename=\"Part1-Subjects1-99.tar\"'), \"Part1-Subjects1-99.tar\")\n    eq_(get_response_disposition_filename('attachment'), None)\n\n\ndef test_parse_url_opts():\n    url = 'http://map.org/api/download/?id=157'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '157'}))\n\n    url = 's3://bucket/save/?key=891'\n    output = parse_url_opts(url)\n    eq_(output, ('s3://bucket/save/', {'key': '891'}))\n\n    url = 'http://map.org/api/download/?id=98&code=13'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '98', 'code': '13'}))\n\n\ndef test_split_colon():\n    eq_(_split_colon('a:b'), ['a', 'b'])\n    eq_(_split_colon('a:b:c'), ['a', 'b:c'])\n    eq_(_split_colon('a:b:c', 2), ['a', 'b', 'c'])\n    eq_(_split_colon('ab'), ['ab'])\n    eq_(_split_colon(r'a\\:b'), [r'a\\:b'])\n\n\ndef test_url_eq():\n    eq_(URL(), URL())\n    neq_(URL(), URL(hostname='x'))\n\n\ndef _check_url(url, **fields):\n    \"\"\"just a helper to carry out few checks on urls\"\"\"\n    url_ = URL(**fields)\n    eq_(URL(url), url_)\n    eq_(str(URL(url)), url)\n    eq_(url, url_)  # just in case ;)  above should fail first if smth is wrong\n    eq_(url, str(url_))  # that we can reconstruct it EXACTLY on our examples\n    # and that we have access to all those fields\n    nok_(set(fields).difference(URL._FIELDS))\n    for f, v in fields.items():\n        eq_(getattr(url_, f), v)\n\n\ndef test_url_base():\n    # Basic checks\n    assert_raises(ValueError, URL, \"http://example.com\", hostname=\"example.com\")\n\n    url = URL(\"http://example.com\")\n    eq_(url.hostname, 'example.com')\n    eq_(url.scheme, 'http')\n    eq_(url.port, '')  # not specified -- empty strings\n    eq_(url.username, '')  # not specified -- empty strings\n    nok_(url.is_implicit)\n    eq_(repr(url), \"URL(hostname='example.com', scheme='http')\")\n    eq_(url, \"http://example.com\")  # automagic coercion in __eq__\n\n    neq_(URL(), URL(hostname='x'))\n\n    smth = URL('smth')\n    eq_(smth.hostname, '')\n    ok_(bool(smth))\n    nok_(bool(URL()))\n\n\ndef test_url_samples():\n    _check_url(\"http://example.com\", scheme='http', hostname=\"example.com\")\n    # \"complete\" one for classical http\n    _check_url(\"http://user:pw@example.com/p/sp?p1=v1&p2=v2#frag\",\n        scheme='http', hostname=\"example.com\", username='user', password='pw', path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n\n    # sample one for ssh with specifying the scheme\n    # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n    _check_url(\"ssh://host/path/sp1\", scheme='ssh', hostname='host', path='/path/sp1')\n    _check_url(\"user@host:path/sp1\",\n               scheme='ssh:implicit', hostname='host', path='path/sp1', username='user')\n    _check_url(\"host:path/sp1\", scheme='ssh:implicit', hostname='host', path='path/sp1')\n    _check_url(\"host:path\", scheme='ssh:implicit', hostname='host', path='path')\n    _check_url(\"host:/path\", scheme='ssh:implicit', hostname='host', path='/path')\n    # TODO!!!  should this be a legit URL like this?\n    # _check_url(\"host\", scheme='ssh:implicit', hostname='host'))\n    eq_(repr(URL(\"host:path\")), \"URL(hostname='host', path='path', scheme='ssh:implicit')\")\n\n    # And now perspective 'datalad:implicit' urls pointing to the canonical center location\n    _check_url(\"///\", scheme='datalad:implicit', path='/')\n    _check_url(\"///p/s1\", scheme='datalad:implicit', path='/p/s1')\n    # could be considered by someone as \"URI reference\" relative to scheme\n    _check_url(\"//a/\", scheme='datalad:implicit', path='/', hostname='a')\n    _check_url(\"//a/data\", scheme='datalad:implicit', path='/data', hostname='a')\n\n    # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n    # or not so custom as\n    _check_url(\"hg+https://host/user/proj\",\n        scheme=\"hg+https\", hostname='host', path='/user/proj')\n    # \"old\" style\n    _check_url(\"dl+archive:KEY/path/sp1#size=123\",\n        scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n    # \"new\" style\n    _check_url(\"dl+archive:KEY#path=path/sp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n    # actually above one is probably wrong since we need to encode the path\n    _check_url(\"dl+archive:KEY#path=path%2Fbsp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n\n    #https://en.wikipedia.org/wiki/File_URI_scheme\n    _check_url(\"file://host\", scheme='file', hostname='host')\n    _check_url(\"file://host/path/sp1\", scheme='file', hostname='host', path='/path/sp1')\n    _check_url(\"file:///path/sp1\", scheme='file', path='/path/sp1')\n    _check_url(\"file:///~/path/sp1\", scheme='file', path='/~/path/sp1')\n    # not sure but let's check\n    _check_url(\"file:///c:/path/sp1\", scheme='file', path='/c:/path/sp1')\n\n    # and now implicit paths or actually they are also \"URI references\"\n    _check_url(\"f\", scheme='file:implicit', path='f')\n    _check_url(\"f/s1\", scheme='file:implicit', path='f/s1')\n    _check_url(\"/f\", scheme='file:implicit', path='/f')\n    _check_url(\"/f/s1\", scheme='file:implicit', path='/f/s1')\n\n    # some github ones, just to make sure\n    _check_url(\"git://host/user/proj\", scheme=\"git\", hostname=\"host\", path=\"/user/proj\")\n    _check_url(\"git@host:user/proj\",\n               scheme=\"ssh:implicit\", hostname=\"host\", path=\"user/proj\", username='git')\n\n    _check_url('weired:/', scheme='ssh:implicit', hostname='weired', path='/')\n    # since schema is not allowing some symbols so we need to add additional check\n    _check_url('weired_url:/', scheme='ssh:implicit', hostname='weired_url', path='/')\n    _check_url('example.com:/', scheme='ssh:implicit', hostname='example.com', path='/')\n    _check_url('example.com:path/sp1', scheme='ssh:implicit', hostname='example.com', path='path/sp1')\n    _check_url('example.com/path/sp1\\:fname',\n               scheme='file:implicit', path='example.com/path/sp1\\:fname')\n    # ssh is as stupid as us, so we will stay \"Consistently\" dumb\n    \"\"\"\n    $> ssh example.com/path/sp1:fname\n    ssh: Could not resolve hostname example.com/path/sp1:fname: Name or service not known\n    \"\"\"\n    _check_url('example.com/path/sp1:fname',\n               scheme='ssh:implicit', hostname='example.com/path/sp1', path='fname')\n\n    # check that we are getting a warning logged when url can't be reconstructed\n    # precisely\n    # actually failed to come up with one -- becomes late here\n    #_check_url(\"http://host///..//p\", scheme='http', path='/..//p')\n\n    # actually this one is good enough to trigger a warning and I still don't know\n    # what it should exactly be!?\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        weired_str = 'weired://'\n        weired_url = URL(weired_str)\n        repr(weired_url)\n        assert_re_in('Parsed version of url .weired. differs from original .weired://.',\n                     cml.out)\n        # but we store original str\n        eq_(str(weired_url), weired_str)\n        neq_(weired_url._as_str(), weired_str)\n\n\ndef test_url_compose_archive_one():\n    url = URL(scheme='dl+archive', path='KEY',\n              fragment=OrderedDict((('path', 'f/p/ s+'), ('size', 30))))\n    # funny - space is encoded as + but + is %2B\n    eq_(str(url), 'dl+archive:KEY#path=f/p/+s%2B&size=30')\n    eq_(url.fragment_dict, {'path': 'f/p/ s+', 'size': '30'})\n\n\ndef test_url_fragments_and_query():\n    url = URL(hostname=\"host\", query=OrderedDict((('a', 'x/b'), ('b', 'y'))))\n    eq_(str(url), '//host?a=x%2Fb&b=y')\n    eq_(url.query, 'a=x%2Fb&b=y')\n    eq_(url.query_dict, {'a': 'x/b', 'b': 'y'})\n\n    url = URL(hostname=\"host\", fragment=OrderedDict((('b', 'x/b'), ('a', 'y'))))\n    eq_(str(url), '//host#b=x/b&a=y')\n    eq_(url.fragment, 'b=x/b&a=y')\n    eq_(url.fragment_dict, {'a': 'y', 'b': 'x/b'})\n\n    fname = get_most_obscure_supported_name()\n    url = URL(hostname=\"host\", fragment={'a': fname})\n    eq_(url.fragment_dict, {'a': fname})\n\n\ndef test_url_dicts():\n    eq_(URL(\"http://host\").query_dict, {})\n\n\n@skip_if_on_windows\ndef test_get_url_path_on_fileurls():\n    eq_(URL('file:///a').path, '/a')\n    eq_(URL('file:///a/b').path, '/a/b')\n    eq_(URL('file:///a/b#id').path, '/a/b')\n    eq_(URL('file:///a/b?whatever').path, '/a/b')\n\n\ndef test_is_url():\n    ok_(is_url('file://localhost/some'))\n    ok_(is_url('http://localhost'))\n    ok_(is_url('ssh://me@localhost'))\n    # in current understanding it is indeed a url but an 'ssh:implicit', not just\n    # a useless scheme=weird with a hope to point to a netloc\n    with swallow_logs():\n        ok_(is_url('weired://'))\n    nok_(is_url('relative'))\n    nok_(is_url('/absolute'))\n    nok_(is_url('like@sshlogin'))\n    nok_(is_url(''))\n    nok_(is_url(' '))\n\n\ndef test_get_local_file_url_linux():\n    eq_(get_local_file_url('/a'), 'file:///a')\n    eq_(get_local_file_url('/a/b/c'), 'file:///a/b/c')\n    eq_(get_local_file_url('/a~'), 'file:///a%7E')\n    eq_(get_local_file_url('/a b/'), 'file:///a%20b/')\n\n\ndef test_get_local_path_from_url():\n    assert_raises(ValueError, get_local_path_from_url, 'http://some')\n    assert_raises(ValueError, get_local_path_from_url, 'file://elsewhere/some')\n    # invalid URL -- is it?  just that 'hostname' is some and no path\n    assert_raises(ValueError, get_local_path_from_url, 'file://some')\n    eq_(get_local_path_from_url('file:///some'), '/some')\n    eq_(get_local_path_from_url('file://localhost/some'), '/some')\n    eq_(get_local_path_from_url('file://::1/some'), '/some')\n    eq_(get_local_path_from_url('file://127.3.4.155/some'), '/some')\n\n\n",
            "header": {
                "file": {
                    "path": "datalad/tests/test_network.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "e415f4e43",
                "new_commit_hash": "1747021cf",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/tests/test_network.py"
            },
            "plus_file": {
                "path": "b/datalad/tests/test_network.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 113,
                            "length": 6
                        },
                        "new_range": {
                            "start": 113,
                            "length": 7
                        },
                        "section": "def _check_url(url, **fields):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    \"\"\"just a helper to carry out few checks on urls\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    url_ = URL(**fields)",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(URL(url), url_)",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(str(URL(url)), url)",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(url, url_)  # just in case ;)  above should fail first if smth is wrong",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(url, str(url_))  # that we can reconstruct it EXACTLY on our examples",
                                "type": "context"
                            },
                            {
                                "content": "    # and that we have access to all those fields",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/tests/test_network.py",
                            "type": "function",
                            "name": "_check_url",
                            "content": "def _check_url(url, **fields):\n    \"\"\"just a helper to carry out few checks on urls\"\"\"\n    url_ = URL(**fields)\n    eq_(URL(url), url_)\n    eq_(str(URL(url)), url)\n    eq_(url, url_)  # just in case ;)  above should fail first if smth is wrong\n    eq_(url, str(url_))  # that we can reconstruct it EXACTLY on our examples\n    # and that we have access to all those fields\n    nok_(set(fields).difference(URL._FIELDS))\n    for f, v in fields.items():\n        eq_(getattr(url_, f), v)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 112,
                            "end_lineno": 122,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 201,
                            "length": 9
                        },
                        "new_range": {
                            "start": 202,
                            "length": 19
                        },
                        "section": "def test_url_samples():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "               scheme=\"ssh:implicit\", hostname=\"host\", path=\"user/proj\", username='git')",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    _check_url('weired:/', scheme='ssh:implicit', hostname='weired', path='/')",
                                "type": "context"
                            },
                            {
                                "content": "    # FAIL?!!!  since schema is not allowing some symbols so we need to add additional",
                                "type": "deleted"
                            },
                            {
                                "content": "    # check",
                                "type": "deleted"
                            },
                            {
                                "content": "    #_check_url('weired_url:/', scheme='ssh:implicit', hostname='weired_url', path='/')",
                                "type": "deleted"
                            },
                            {
                                "content": "    # since schema is not allowing some symbols so we need to add additional check",
                                "type": "added"
                            },
                            {
                                "content": "    _check_url('weired_url:/', scheme='ssh:implicit', hostname='weired_url', path='/')",
                                "type": "added"
                            },
                            {
                                "content": "    _check_url('example.com:/', scheme='ssh:implicit', hostname='example.com', path='/')",
                                "type": "added"
                            },
                            {
                                "content": "    _check_url('example.com:path/sp1', scheme='ssh:implicit', hostname='example.com', path='path/sp1')",
                                "type": "added"
                            },
                            {
                                "content": "    _check_url('example.com/path/sp1\\:fname',",
                                "type": "added"
                            },
                            {
                                "content": "               scheme='file:implicit', path='example.com/path/sp1\\:fname')",
                                "type": "added"
                            },
                            {
                                "content": "    # ssh is as stupid as us, so we will stay \"Consistently\" dumb",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "    $> ssh example.com/path/sp1:fname",
                                "type": "added"
                            },
                            {
                                "content": "    ssh: Could not resolve hostname example.com/path/sp1:fname: Name or service not known",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "    _check_url('example.com/path/sp1:fname',",
                                "type": "added"
                            },
                            {
                                "content": "               scheme='ssh:implicit', hostname='example.com/path/sp1', path='fname')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # check that we are getting a warning logged when url can't be reconstructed",
                                "type": "context"
                            },
                            {
                                "content": "    # precisely",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/tests/test_network.py",
                            "type": "function",
                            "name": "test_url_samples",
                            "content": "def test_url_samples():\n    _check_url(\"http://example.com\", scheme='http', hostname=\"example.com\")\n    # \"complete\" one for classical http\n    _check_url(\"http://user:pw@example.com/p/sp?p1=v1&p2=v2#frag\",\n        scheme='http', hostname=\"example.com\", username='user', password='pw', path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n\n    # sample one for ssh with specifying the scheme\n    # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n    _check_url(\"ssh://host/path/sp1\", scheme='ssh', hostname='host', path='/path/sp1')\n    _check_url(\"user@host:path/sp1\",\n               scheme='ssh:implicit', hostname='host', path='path/sp1', username='user')\n    _check_url(\"host:path/sp1\", scheme='ssh:implicit', hostname='host', path='path/sp1')\n    _check_url(\"host:path\", scheme='ssh:implicit', hostname='host', path='path')\n    _check_url(\"host:/path\", scheme='ssh:implicit', hostname='host', path='/path')\n    # TODO!!!  should this be a legit URL like this?\n    # _check_url(\"host\", scheme='ssh:implicit', hostname='host'))\n    eq_(repr(URL(\"host:path\")), \"URL(hostname='host', path='path', scheme='ssh:implicit')\")\n\n    # And now perspective 'datalad:implicit' urls pointing to the canonical center location\n    _check_url(\"///\", scheme='datalad:implicit', path='/')\n    _check_url(\"///p/s1\", scheme='datalad:implicit', path='/p/s1')\n    # could be considered by someone as \"URI reference\" relative to scheme\n    _check_url(\"//a/\", scheme='datalad:implicit', path='/', hostname='a')\n    _check_url(\"//a/data\", scheme='datalad:implicit', path='/data', hostname='a')\n\n    # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n    # or not so custom as\n    _check_url(\"hg+https://host/user/proj\",\n        scheme=\"hg+https\", hostname='host', path='/user/proj')\n    # \"old\" style\n    _check_url(\"dl+archive:KEY/path/sp1#size=123\",\n        scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n    # \"new\" style\n    _check_url(\"dl+archive:KEY#path=path/sp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n    # actually above one is probably wrong since we need to encode the path\n    _check_url(\"dl+archive:KEY#path=path%2Fbsp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n\n    #https://en.wikipedia.org/wiki/File_URI_scheme\n    _check_url(\"file://host\", scheme='file', hostname='host')\n    _check_url(\"file://host/path/sp1\", scheme='file', hostname='host', path='/path/sp1')\n    _check_url(\"file:///path/sp1\", scheme='file', path='/path/sp1')\n    _check_url(\"file:///~/path/sp1\", scheme='file', path='/~/path/sp1')\n    # not sure but let's check\n    _check_url(\"file:///c:/path/sp1\", scheme='file', path='/c:/path/sp1')\n\n    # and now implicit paths or actually they are also \"URI references\"\n    _check_url(\"f\", scheme='file:implicit', path='f')\n    _check_url(\"f/s1\", scheme='file:implicit', path='f/s1')\n    _check_url(\"/f\", scheme='file:implicit', path='/f')\n    _check_url(\"/f/s1\", scheme='file:implicit', path='/f/s1')\n\n    # some github ones, just to make sure\n    _check_url(\"git://host/user/proj\", scheme=\"git\", hostname=\"host\", path=\"/user/proj\")\n    _check_url(\"git@host:user/proj\",\n               scheme=\"ssh:implicit\", hostname=\"host\", path=\"user/proj\", username='git')\n\n    _check_url('weired:/', scheme='ssh:implicit', hostname='weired', path='/')\n    # FAIL?!!!  since schema is not allowing some symbols so we need to add additional\n    # check\n    #_check_url('weired_url:/', scheme='ssh:implicit', hostname='weired_url', path='/')\n\n    # check that we are getting a warning logged when url can't be reconstructed\n    # precisely\n    # actually failed to come up with one -- becomes late here\n    #_check_url(\"http://host///..//p\", scheme='http', path='/..//p')\n\n    # actually this one is good enough to trigger a warning and I still don't know\n    # what it should exactly be!?\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        weired_str = 'weired://'\n        weired_url = URL(weired_str)\n        repr(weired_url)\n        assert_re_in('Parsed version of url .weired. differs from original .weired://.',\n                     cml.out)\n        # but we store original str\n        eq_(str(weired_url), weired_str)\n        neq_(weired_url._as_str(), weired_str)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 145,
                            "end_lineno": 223,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "6b6fa3898546793fa3517def09f85a74d51ec4bb^",
    "new_commit_hash": "6b6fa3898546793fa3517def09f85a74d51ec4bb",
    "commit_message": "BF: additional pattern for ssh:implicit -- forcefully staying as dumb as openssh client for now",
    "commit_date": "2016-06-05T14:21:42-04:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "6b6fa3898546793fa3517def09f85a74d51ec4bb",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport logging\n\nfrom collections import OrderedDict\n\nfrom datalad.tests.utils import eq_, neq_, ok_, nok_, assert_raises\nfrom datalad.tests.utils import skip_if_on_windows\nfrom datalad.tests.utils import swallow_logs\nfrom datalad.tests.utils import assert_re_in\nfrom datalad.tests.utils import get_most_obscure_supported_name\n\nfrom datalad.support.network import same_website, dlurljoin\nfrom datalad.support.network import get_tld\nfrom datalad.support.network import get_url_straight_filename\nfrom datalad.support.network import get_response_disposition_filename\nfrom datalad.support.network import parse_url_opts\nfrom datalad.support.network import URL\nfrom datalad.support.network import _split_colon\nfrom datalad.support.network import is_url\nfrom datalad.support.network import get_local_file_url\nfrom datalad.support.network import get_local_path_from_url\n\ndef test_same_website():\n    ok_(same_website(\"http://a.b\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"https://a.b/page/2/\", \"http://a.b/2014/01/xxx/\"))\n    ok_(same_website(\"http://a.b/page/2/\", \"https://a.b/2014/01/xxx/\"))\n\n\ndef test_get_tld():\n    eq_(get_tld('http://example.com'), 'example.com')\n    eq_(get_tld('http://example.com/1'), 'example.com')\n    eq_(get_tld('http://example.com/1/2'), 'example.com')\n    eq_(get_tld('example.com/1/2'), 'example.com')\n    eq_(get_tld('s3://example.com/1/2'), 'example.com')\n    assert_raises(ValueError, get_tld, \"\")\n    assert_raises(ValueError, get_tld, \"s3://\")\n    assert_raises(ValueError, get_tld, \"http://\")\n\ndef test_dlurljoin():\n    eq_(dlurljoin('http://a.b/', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/page', 'f'), 'http://a.b/f')\n    eq_(dlurljoin('http://a.b/dir/', 'f'), 'http://a.b/dir/f')\n    eq_(dlurljoin('http://a.b/dir/', 'http://url'), 'http://url')\n    eq_(dlurljoin('http://a.b/dir/', '/'), 'http://a.b/')\n    eq_(dlurljoin('http://a.b/dir/', '/x/y'), 'http://a.b/x/y')\n\ndef _test_get_url_straight_filename(suf):\n    eq_(get_url_straight_filename('http://a.b/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1' + suf), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/' + suf, allowdir=True), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2' + suf), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf), '')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True), 'p2')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, allowdir=True, strip=('p2', 'xxx')), 'p1')\n    eq_(get_url_straight_filename('http://a.b/p1/p2/' + suf, strip=('p2', 'xxx')), '')\n\ndef test_get_url_straight_filename():\n    yield _test_get_url_straight_filename, ''\n    yield _test_get_url_straight_filename, '#'\n    yield _test_get_url_straight_filename, '#tag'\n    yield _test_get_url_straight_filename, '#tag/obscure'\n    yield _test_get_url_straight_filename, '?param=1'\n    yield _test_get_url_straight_filename, '?param=1&another=/'\n\nfrom datalad.support.network import rfc2822_to_epoch\ndef test_rfc2822_to_epoch():\n    eq_(rfc2822_to_epoch(\"Thu, 16 Oct 2014 01:16:17 EDT\"), 1413436577)\n\n\ndef test_get_response_disposition_filename():\n    eq_(get_response_disposition_filename('attachment;filename=\"Part1-Subjects1-99.tar\"'), \"Part1-Subjects1-99.tar\")\n    eq_(get_response_disposition_filename('attachment'), None)\n\n\ndef test_parse_url_opts():\n    url = 'http://map.org/api/download/?id=157'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '157'}))\n\n    url = 's3://bucket/save/?key=891'\n    output = parse_url_opts(url)\n    eq_(output, ('s3://bucket/save/', {'key': '891'}))\n\n    url = 'http://map.org/api/download/?id=98&code=13'\n    output = parse_url_opts(url)\n    eq_(output, ('http://map.org/api/download/', {'id': '98', 'code': '13'}))\n\n\ndef test_split_colon():\n    eq_(_split_colon('a:b'), ['a', 'b'])\n    eq_(_split_colon('a:b:c'), ['a', 'b:c'])\n    eq_(_split_colon('a:b:c', 2), ['a', 'b', 'c'])\n    eq_(_split_colon('ab'), ['ab'])\n    eq_(_split_colon(r'a\\:b'), [r'a\\:b'])\n\n\ndef test_url_eq():\n    eq_(URL(), URL())\n    neq_(URL(), URL(hostname='x'))\n\n\ndef _check_url(url, **fields):\n    \"\"\"just a helper to carry out few checks on urls\"\"\"\n    url_ = URL(**fields)\n    eq_(URL(url), url_)\n    eq_(str(URL(url)), url)\n    eq_(url, url_)  # just in case ;)  above should fail first if smth is wrong\n    eq_(url, str(url_))  # that we can reconstruct it EXACTLY on our examples\n    # and that we have access to all those fields\n    nok_(set(fields).difference(URL._FIELDS))\n    for f, v in fields.items():\n        eq_(getattr(url_, f), v)\n\n\ndef test_url_base():\n    # Basic checks\n    assert_raises(ValueError, URL, \"http://example.com\", hostname=\"example.com\")\n\n    url = URL(\"http://example.com\")\n    eq_(url.hostname, 'example.com')\n    eq_(url.scheme, 'http')\n    eq_(url.port, '')  # not specified -- empty strings\n    eq_(url.username, '')  # not specified -- empty strings\n    nok_(url.is_implicit)\n    eq_(repr(url), \"URL(hostname='example.com', scheme='http')\")\n    eq_(url, \"http://example.com\")  # automagic coercion in __eq__\n\n    neq_(URL(), URL(hostname='x'))\n\n    smth = URL('smth')\n    eq_(smth.hostname, '')\n    ok_(bool(smth))\n    nok_(bool(URL()))\n\n\ndef test_url_samples():\n    _check_url(\"http://example.com\", scheme='http', hostname=\"example.com\")\n    # \"complete\" one for classical http\n    _check_url(\"http://user:pw@example.com/p/sp?p1=v1&p2=v2#frag\",\n        scheme='http', hostname=\"example.com\", username='user', password='pw', path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n\n    # sample one for ssh with specifying the scheme\n    # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n    _check_url(\"ssh://host/path/sp1\", scheme='ssh', hostname='host', path='/path/sp1')\n    _check_url(\"user@host:path/sp1\",\n               scheme='ssh:implicit', hostname='host', path='path/sp1', username='user')\n    _check_url(\"host:path/sp1\", scheme='ssh:implicit', hostname='host', path='path/sp1')\n    _check_url(\"host:path\", scheme='ssh:implicit', hostname='host', path='path')\n    _check_url(\"host:/path\", scheme='ssh:implicit', hostname='host', path='/path')\n    # TODO!!!  should this be a legit URL like this?\n    # _check_url(\"host\", scheme='ssh:implicit', hostname='host'))\n    eq_(repr(URL(\"host:path\")), \"URL(hostname='host', path='path', scheme='ssh:implicit')\")\n\n    # And now perspective 'datalad:implicit' urls pointing to the canonical center location\n    _check_url(\"///\", scheme='datalad:implicit', path='/')\n    _check_url(\"///p/s1\", scheme='datalad:implicit', path='/p/s1')\n    # could be considered by someone as \"URI reference\" relative to scheme\n    _check_url(\"//a/\", scheme='datalad:implicit', path='/', hostname='a')\n    _check_url(\"//a/data\", scheme='datalad:implicit', path='/data', hostname='a')\n\n    # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n    # or not so custom as\n    _check_url(\"hg+https://host/user/proj\",\n        scheme=\"hg+https\", hostname='host', path='/user/proj')\n    # \"old\" style\n    _check_url(\"dl+archive:KEY/path/sp1#size=123\",\n        scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n    # \"new\" style\n    _check_url(\"dl+archive:KEY#path=path/sp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n    # actually above one is probably wrong since we need to encode the path\n    _check_url(\"dl+archive:KEY#path=path%2Fbsp1&size=123\",\n        scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n\n    #https://en.wikipedia.org/wiki/File_URI_scheme\n    _check_url(\"file://host\", scheme='file', hostname='host')\n    _check_url(\"file://host/path/sp1\", scheme='file', hostname='host', path='/path/sp1')\n    _check_url(\"file:///path/sp1\", scheme='file', path='/path/sp1')\n    _check_url(\"file:///~/path/sp1\", scheme='file', path='/~/path/sp1')\n    # not sure but let's check\n    _check_url(\"file:///c:/path/sp1\", scheme='file', path='/c:/path/sp1')\n\n    # and now implicit paths or actually they are also \"URI references\"\n    _check_url(\"f\", scheme='file:implicit', path='f')\n    _check_url(\"f/s1\", scheme='file:implicit', path='f/s1')\n    _check_url(\"/f\", scheme='file:implicit', path='/f')\n    _check_url(\"/f/s1\", scheme='file:implicit', path='/f/s1')\n\n    # some github ones, just to make sure\n    _check_url(\"git://host/user/proj\", scheme=\"git\", hostname=\"host\", path=\"/user/proj\")\n    _check_url(\"git@host:user/proj\",\n               scheme=\"ssh:implicit\", hostname=\"host\", path=\"user/proj\", username='git')\n\n    _check_url('weired:/', scheme='ssh:implicit', hostname='weired', path='/')\n    # since schema is not allowing some symbols so we need to add additional check\n    _check_url('weired_url:/', scheme='ssh:implicit', hostname='weired_url', path='/')\n    _check_url('example.com:/', scheme='ssh:implicit', hostname='example.com', path='/')\n    _check_url('example.com:path/sp1', scheme='ssh:implicit', hostname='example.com', path='path/sp1')\n    _check_url('example.com/path/sp1\\:fname',\n               scheme='file:implicit', path='example.com/path/sp1\\:fname')\n    # ssh is as stupid as us, so we will stay \"Consistently\" dumb\n    \"\"\"\n    $> ssh example.com/path/sp1:fname\n    ssh: Could not resolve hostname example.com/path/sp1:fname: Name or service not known\n    \"\"\"\n    _check_url('example.com/path/sp1:fname',\n               scheme='ssh:implicit', hostname='example.com/path/sp1', path='fname')\n\n    # check that we are getting a warning logged when url can't be reconstructed\n    # precisely\n    # actually failed to come up with one -- becomes late here\n    #_check_url(\"http://host///..//p\", scheme='http', path='/..//p')\n\n    # actually this one is good enough to trigger a warning and I still don't know\n    # what it should exactly be!?\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        weired_str = 'weired://'\n        weired_url = URL(weired_str)\n        repr(weired_url)\n        assert_re_in('Parsed version of url .weired. differs from original .weired://.',\n                     cml.out)\n        # but we store original str\n        eq_(str(weired_url), weired_str)\n        neq_(weired_url._as_str(), weired_str)\n\n\ndef test_url_compose_archive_one():\n    url = URL(scheme='dl+archive', path='KEY',\n              fragment=OrderedDict((('path', 'f/p/ s+'), ('size', 30))))\n    # funny - space is encoded as + but + is %2B\n    eq_(str(url), 'dl+archive:KEY#path=f/p/+s%2B&size=30')\n    eq_(url.fragment_dict, {'path': 'f/p/ s+', 'size': '30'})\n\n\ndef test_url_fragments_and_query():\n    url = URL(hostname=\"host\", query=OrderedDict((('a', 'x/b'), ('b', 'y'))))\n    eq_(str(url), '//host?a=x%2Fb&b=y')\n    eq_(url.query, 'a=x%2Fb&b=y')\n    eq_(url.query_dict, {'a': 'x/b', 'b': 'y'})\n\n    url = URL(hostname=\"host\", fragment=OrderedDict((('b', 'x/b'), ('a', 'y'))))\n    eq_(str(url), '//host#b=x/b&a=y')\n    eq_(url.fragment, 'b=x/b&a=y')\n    eq_(url.fragment_dict, {'a': 'y', 'b': 'x/b'})\n\n    fname = get_most_obscure_supported_name()\n    url = URL(hostname=\"host\", fragment={'a': fname})\n    eq_(url.fragment_dict, {'a': fname})\n\n\ndef test_url_dicts():\n    eq_(URL(\"http://host\").query_dict, {})\n\n\n@skip_if_on_windows\ndef test_get_url_path_on_fileurls():\n    eq_(URL('file:///a').path, '/a')\n    eq_(URL('file:///a/b').path, '/a/b')\n    eq_(URL('file:///a/b#id').path, '/a/b')\n    eq_(URL('file:///a/b?whatever').path, '/a/b')\n\n\ndef test_is_url():\n    ok_(is_url('file://localhost/some'))\n    ok_(is_url('http://localhost'))\n    ok_(is_url('ssh://me@localhost'))\n    # in current understanding it is indeed a url but an 'ssh:implicit', not just\n    # a useless scheme=weird with a hope to point to a netloc\n    with swallow_logs():\n        ok_(is_url('weired://'))\n    nok_(is_url('relative'))\n    nok_(is_url('/absolute'))\n    nok_(is_url('like@sshlogin'))\n    nok_(is_url(''))\n    nok_(is_url(' '))\n\n\ndef test_get_local_file_url_linux():\n    eq_(get_local_file_url('/a'), 'file:///a')\n    eq_(get_local_file_url('/a/b/c'), 'file:///a/b/c')\n    eq_(get_local_file_url('/a~'), 'file:///a%7E')\n    eq_(get_local_file_url('/a b/'), 'file:///a%20b/')\n\n\ndef test_get_local_path_from_url():\n    assert_raises(ValueError, get_local_path_from_url, 'http://some')\n    assert_raises(ValueError, get_local_path_from_url, 'file://elsewhere/some')\n    # invalid URL -- is it?  just that 'hostname' is some and no path\n    assert_raises(ValueError, get_local_path_from_url, 'file://some')\n    eq_(get_local_path_from_url('file:///some'), '/some')\n    eq_(get_local_path_from_url('file://localhost/some'), '/some')\n    eq_(get_local_path_from_url('file://::1/some'), '/some')\n    eq_(get_local_path_from_url('file://127.3.4.155/some'), '/some')\n\n\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.2.dev191\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 13ms\nInstalled 25 packages in 2.35s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptkyOkg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptkyOkg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptkyOkg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptkyOkg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmptkyOkg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 35, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWjzklv/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWjzklv/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWjzklv/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWjzklv/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWjzklv/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 35, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6b6fa3898546793fa3517def09f85a74d51ec4bb/datalad/__init__.py\", line 20, in <module>\n    from .config import ConfigManager\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6b6fa3898546793fa3517def09f85a74d51ec4bb/datalad/config.py\", line 17, in <module>\n    from appdirs import AppDirs\nModuleNotFoundError: No module named 'appdirs'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 213ms\nInstalled 26 packages in 4.77s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 67 packages in 4.13s\nPrepared 1 package in 4.49s\nInstalled 61 packages in 756ms\n + appdirs==1.4.4\n + attrs==24.2.0\n + automat==22.10.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + charset-normalizer==3.4.1\n + constantly==15.1.0\n + cryptography==44.0.0\n + cssselect==1.2.0\n + datalad==0.2.dev1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6b6fa3898546793fa3517def09f85a74d51ec4bb)\n + filelock==3.12.2\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + hyperlink==21.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + incremental==22.10.0\n + itemadapter==0.8.0\n + itemloaders==1.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jmespath==1.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack-python==0.5.6\n + multidict==6.0.5\n + nose==1.3.7\n + parsel==1.8.1\n + patool==1.12\n + protego==0.3.0\n + pyasn1==0.5.1\n + pyasn1-modules==0.3.0\n + pycparser==2.21\n + pydispatcher==2.0.7\n + pyopenssl==24.3.0\n + pyyaml==6.0.1\n + queuelib==1.6.2\n + requests==2.31.0\n + requests-file==2.1.0\n + scrapy==2.9.0\n + secretstorage==3.3.3\n + service-identity==21.1.0\n + smmap==5.0.1\n + soupsieve==2.4.1\n + testtools==2.7.1\n + tldextract==4.0.0\n + tqdm==4.67.1\n + twisted==23.8.0\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + w3lib==2.1.2\n + wrapt==1.16.0\n + yarl==1.9.4\n + zope-interface==6.4.post2\nAudited 1 package in 20ms\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6b6fa3898546793fa3517def09f85a74d51ec4bb\nplugins: cov-4.1.0\ncollected 18 items\n\nr2e_tests/test_1.py ...x............F.                                   [100%]\n\n=================================== FAILURES ===================================\n________________________ test_get_local_file_url_linux _________________________\n\n    def test_get_local_file_url_linux():\n        eq_(get_local_file_url('/a'), 'file:///a')\n        eq_(get_local_file_url('/a/b/c'), 'file:///a/b/c')\n>       eq_(get_local_file_url('/a~'), 'file:///a%7E')\n\nr2e_tests/test_1.py:291: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = 'file:///a~', b = 'file:///a%7E', msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: 'file:///a~' != 'file:///a%7E'\n\n.venv/lib/python3.7/site-packages/nose/tools/trivial.py:29: AssertionError\n==================================== PASSES ====================================\n_______________________________ test_url_samples _______________________________\n------------------------------ Captured log call -------------------------------\nWARNING  datalad.network:network.py:539 Parsed version of url 'weired' differs from original 'weired://'\n_________________________________ test_is_url __________________________________\n------------------------------ Captured log call -------------------------------\nWARNING  datalad.network:network.py:539 Parsed version of url 'weired' differs from original 'weired://'\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_same_website\nPASSED r2e_tests/test_1.py::test_get_tld\nPASSED r2e_tests/test_1.py::test_dlurljoin\nPASSED r2e_tests/test_1.py::test_rfc2822_to_epoch\nPASSED r2e_tests/test_1.py::test_get_response_disposition_filename\nPASSED r2e_tests/test_1.py::test_parse_url_opts\nPASSED r2e_tests/test_1.py::test_split_colon\nPASSED r2e_tests/test_1.py::test_url_eq\nPASSED r2e_tests/test_1.py::test_url_base\nPASSED r2e_tests/test_1.py::test_url_samples\nPASSED r2e_tests/test_1.py::test_url_compose_archive_one\nPASSED r2e_tests/test_1.py::test_url_fragments_and_query\nPASSED r2e_tests/test_1.py::test_url_dicts\nPASSED r2e_tests/test_1.py::test_get_url_path_on_fileurls\nPASSED r2e_tests/test_1.py::test_is_url\nPASSED r2e_tests/test_1.py::test_get_local_path_from_url\nXFAIL r2e_tests/test_1.py::test_get_url_straight_filename - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_get_url_straight_filename will be ignored\nFAILED r2e_tests/test_1.py::test_get_local_file_url_linux - AssertionError: '...\n=================== 1 failed, 16 passed, 1 xfailed in 1.65s ====================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6b6fa3898546793fa3517def09f85a74d51ec4bb\nplugins: cov-4.1.0\ncollected 18 items\n\nr2e_tests/test_1.py ...x......F.....F.                                   [100%]\n\n=================================== FAILURES ===================================\n_______________________________ test_url_samples _______________________________\n\n    def test_url_samples():\n        _check_url(\"http://example.com\", scheme='http', hostname=\"example.com\")\n        # \"complete\" one for classical http\n        _check_url(\"http://user:pw@example.com/p/sp?p1=v1&p2=v2#frag\",\n            scheme='http', hostname=\"example.com\", username='user', password='pw', path='/p/sp', query='p1=v1&p2=v2', fragment='frag')\n    \n        # sample one for ssh with specifying the scheme\n        # XXX? might be useful?  https://github.com/FriendCode/giturlparse.py\n        _check_url(\"ssh://host/path/sp1\", scheme='ssh', hostname='host', path='/path/sp1')\n        _check_url(\"user@host:path/sp1\",\n                   scheme='ssh:implicit', hostname='host', path='path/sp1', username='user')\n        _check_url(\"host:path/sp1\", scheme='ssh:implicit', hostname='host', path='path/sp1')\n        _check_url(\"host:path\", scheme='ssh:implicit', hostname='host', path='path')\n        _check_url(\"host:/path\", scheme='ssh:implicit', hostname='host', path='/path')\n        # TODO!!!  should this be a legit URL like this?\n        # _check_url(\"host\", scheme='ssh:implicit', hostname='host'))\n        eq_(repr(URL(\"host:path\")), \"URL(hostname='host', path='path', scheme='ssh:implicit')\")\n    \n        # And now perspective 'datalad:implicit' urls pointing to the canonical center location\n        _check_url(\"///\", scheme='datalad:implicit', path='/')\n        _check_url(\"///p/s1\", scheme='datalad:implicit', path='/p/s1')\n        # could be considered by someone as \"URI reference\" relative to scheme\n        _check_url(\"//a/\", scheme='datalad:implicit', path='/', hostname='a')\n        _check_url(\"//a/data\", scheme='datalad:implicit', path='/data', hostname='a')\n    \n        # here we will do custom magic allowing only schemes with + in them, such as dl+archive\n        # or not so custom as\n        _check_url(\"hg+https://host/user/proj\",\n            scheme=\"hg+https\", hostname='host', path='/user/proj')\n        # \"old\" style\n        _check_url(\"dl+archive:KEY/path/sp1#size=123\",\n            scheme='dl+archive', path='KEY/path/sp1', fragment='size=123')\n        # \"new\" style\n        _check_url(\"dl+archive:KEY#path=path/sp1&size=123\",\n            scheme='dl+archive', path='KEY', fragment='path=path/sp1&size=123')\n        # actually above one is probably wrong since we need to encode the path\n        _check_url(\"dl+archive:KEY#path=path%2Fbsp1&size=123\",\n            scheme='dl+archive', path='KEY', fragment='path=path%2Fbsp1&size=123')\n    \n        #https://en.wikipedia.org/wiki/File_URI_scheme\n        _check_url(\"file://host\", scheme='file', hostname='host')\n        _check_url(\"file://host/path/sp1\", scheme='file', hostname='host', path='/path/sp1')\n        _check_url(\"file:///path/sp1\", scheme='file', path='/path/sp1')\n        _check_url(\"file:///~/path/sp1\", scheme='file', path='/~/path/sp1')\n        # not sure but let's check\n        _check_url(\"file:///c:/path/sp1\", scheme='file', path='/c:/path/sp1')\n    \n        # and now implicit paths or actually they are also \"URI references\"\n        _check_url(\"f\", scheme='file:implicit', path='f')\n        _check_url(\"f/s1\", scheme='file:implicit', path='f/s1')\n        _check_url(\"/f\", scheme='file:implicit', path='/f')\n        _check_url(\"/f/s1\", scheme='file:implicit', path='/f/s1')\n    \n        # some github ones, just to make sure\n        _check_url(\"git://host/user/proj\", scheme=\"git\", hostname=\"host\", path=\"/user/proj\")\n        _check_url(\"git@host:user/proj\",\n                   scheme=\"ssh:implicit\", hostname=\"host\", path=\"user/proj\", username='git')\n    \n        _check_url('weired:/', scheme='ssh:implicit', hostname='weired', path='/')\n        # since schema is not allowing some symbols so we need to add additional check\n>       _check_url('weired_url:/', scheme='ssh:implicit', hostname='weired_url', path='/')\n\nr2e_tests/test_1.py:206: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \nr2e_tests/test_1.py:115: in _check_url\n    eq_(URL(url), url_)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = URL(path='weired_url:/', scheme='file:implicit')\nb = URL(hostname='weired_url', path='/', scheme='ssh:implicit'), msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: URL(path='weired_url:/', scheme='file:implicit') != URL(hostname='weired_url', path='/', scheme='ssh:implicit')\n\n.venv/lib/python3.7/site-packages/nose/tools/trivial.py:29: AssertionError\n________________________ test_get_local_file_url_linux _________________________\n\n    def test_get_local_file_url_linux():\n        eq_(get_local_file_url('/a'), 'file:///a')\n        eq_(get_local_file_url('/a/b/c'), 'file:///a/b/c')\n>       eq_(get_local_file_url('/a~'), 'file:///a%7E')\n\nr2e_tests/test_1.py:291: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = 'file:///a~', b = 'file:///a%7E', msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: 'file:///a~' != 'file:///a%7E'\n\n.venv/lib/python3.7/site-packages/nose/tools/trivial.py:29: AssertionError\n==================================== PASSES ====================================\n_________________________________ test_is_url __________________________________\n------------------------------ Captured log call -------------------------------\nWARNING  datalad.network:network.py:531 Parsed version of url 'weired' differs from original 'weired://'\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_same_website\nPASSED r2e_tests/test_1.py::test_get_tld\nPASSED r2e_tests/test_1.py::test_dlurljoin\nPASSED r2e_tests/test_1.py::test_rfc2822_to_epoch\nPASSED r2e_tests/test_1.py::test_get_response_disposition_filename\nPASSED r2e_tests/test_1.py::test_parse_url_opts\nPASSED r2e_tests/test_1.py::test_split_colon\nPASSED r2e_tests/test_1.py::test_url_eq\nPASSED r2e_tests/test_1.py::test_url_base\nPASSED r2e_tests/test_1.py::test_url_compose_archive_one\nPASSED r2e_tests/test_1.py::test_url_fragments_and_query\nPASSED r2e_tests/test_1.py::test_url_dicts\nPASSED r2e_tests/test_1.py::test_get_url_path_on_fileurls\nPASSED r2e_tests/test_1.py::test_is_url\nPASSED r2e_tests/test_1.py::test_get_local_path_from_url\nXFAIL r2e_tests/test_1.py::test_get_url_straight_filename - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_get_url_straight_filename will be ignored\nFAILED r2e_tests/test_1.py::test_url_samples - AssertionError: URL(path='weir...\nFAILED r2e_tests/test_1.py::test_get_local_file_url_linux - AssertionError: '...\n=================== 2 failed, 15 passed, 1 xfailed in 0.87s ====================\n",
    "old_commit_res_stderr": ""
}
datalad/support/network.py datalad/tests/test_network.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Incorrect Parsing of SSH Implicit URLs Causes Parsing Failures

**Description:**
When parsing SSH implicit URLs that include a hostname followed by a colon and a path, the URL is incorrectly interpreted using the `file:implicit` scheme instead of `ssh:implicit`. This leads to incorrect separation of the hostname and path components, causing downstream operations to fail.

**Example Code:**
```python
from datalad.support.network import URL

url = 'weired_url:/'
parsed_url = URL(url)

print(parsed_url.scheme)      # Expected: 'ssh:implicit'
print(parsed_url.hostname)   # Expected: 'weired_url'
print(parsed_url.path)       # Expected: '/'
```

**Expected Behavior:**
The URL should be parsed with the `ssh:implicit` scheme, correctly identifying the hostname as `'weired_url'` and the path as `'/'`.

**Actual Behavior:**
The URL is parsed with the `file:implicit` scheme, resulting in the entire string `'weired_url:/'` being set as the path and leaving the hostname undefined. This mismatch causes assertions and operations that rely on proper URL parsing to fail.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/network.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

import calendar
import email.utils
import gzip
import os
import re
import shutil
import time

from collections import OrderedDict
from os.path import abspath, isabs

from six import string_types
from six import iteritems
from six.moves.urllib.request import urlopen, Request
from six.moves.urllib.parse import quote as urlquote, unquote as urlunquote
from six.moves.urllib.parse import urljoin, urlparse, urlsplit, urlunsplit, urlunparse, ParseResult
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlencode
from six.moves.urllib.error import URLError

from datalad.utils import on_windows

from datalad.utils import auto_repr
# TODO not sure what needs to use `six` here yet
import requests

import logging
lgr = logging.getLogger('datalad.network')


def get_response_disposition_filename(s):
    """Given a string s as from HTTP Content-Disposition field in the response
    return possibly present filename if any
    """
    if not s:
        return None
    # If the response has Content-Disposition, try to get filename from it
    cd = dict(map(
        lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),
        s.split(';')))
    if 'filename' in cd:
        filename = cd['filename'].strip("\"'")
        return filename
    return None


def get_url_disposition_filename(url, headers=None):
    """Get filename as possibly provided by the server in Content-Disposition
    """
    if headers is None:
        request = Request(url)
        r = retry_urlopen(request)
        # things are different in requests
        if 'requests.' in str(r.__class__):
            headers = r.headers
        else:
            headers = r.info()
    else:
        r = None
    try:
        return get_response_disposition_filename(headers.get('Content-Disposition', ''))
    finally:
        if r:
            r.close()


def get_url_straight_filename(url, strip=[], allowdir=False):
    """Get file/dir name of the last path component of the URL

    Parameters
    ----------
    strip: list, optional
      If provided, listed names will not be considered and their
      parent directory will be selected
    allowdir: bool, optional
      If url points to a "directory" (ends with /), empty string
      would be returned unless allowdir is True, in which case the
      name of the directory would be returned
    """
    path = urlunquote(urlsplit(url).path)
    path_parts = path.split('/')

    if allowdir:
        # strip empty ones
        while len(path_parts) > 1 and not path_parts[-1]:
            path_parts = path_parts[:-1]

    if strip:
        while path_parts and path_parts[-1] in strip:
            path_parts = path_parts[:-1]

    if path_parts:
        return path_parts[-1]
    else:
        return None


def get_url_filename(url, headers=None, strip=[]):
    """Get filename from the url, first consulting server about Content-Disposition
    """
    filename = get_url_disposition_filename(url, headers)
    if filename:
        return filename
    return get_url_straight_filename(url, strip=[])


def get_url_response_stamp(url, response_info):
    size, mtime = None, None
    if 'Content-length' in response_info:
        size = int(response_info['Content-length'])
    if 'Last-modified' in response_info:
        mtime = calendar.timegm(email.utils.parsedate(
            response_info['Last-modified']))
    return dict(size=size, mtime=mtime, url=url)


def get_tld(url):
    """Return top level domain from a url

    Parameters
    ----------
    url : str
    """
    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld
    if not url.strip():
        raise ValueError("Empty URL has no TLD")
    rec = urlsplit(url)
    if not rec.netloc:
        if not rec.scheme:
            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'
            return url.split('/', 1)[0]
        else:
            raise ValueError("It seems that only the scheme was provided without the net location/TLD")
    return rec.netloc


from email.utils import parsedate_tz, mktime_tz
def rfc2822_to_epoch(datestr):
    """Given rfc2822 date/time format, return seconds since epoch"""
    return mktime_tz(parsedate_tz(datestr))


import calendar
from datetime import datetime
def iso8601_to_epoch(datestr):
    return calendar.timegm(datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%S.%fZ").timetuple())


def __urlopen_requests(url, header_vals=None):
    # XXX Workaround for now for ... broken code
    if isinstance(url, Request):
        url = url.get_full_url()
    return requests.Session().get(url)


def retry_urlopen(url, retries=3):
    for t in range(retries):
        try:
            return __urlopen_requests(url)
        except URLError as e:
            lgr.warn("Received exception while reading %s: %s" % (url, e))
            if t == retries - 1:
                # if we have reached allowed number of retries -- reraise
                raise


def is_url_quoted(url):
    """Return either URL looks being already quoted
    """
    try:
        url_ = urlunquote(url)
        return url != url_
    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)
        return False


def same_website(url_rec, u_rec):
    """Decide either a link leads to external site

    Parameters
    ----------
    url_rec: ParseResult
      record for original url
    u_rec: ParseResult
      record for new url
    """
    if isinstance(url_rec, string_types):
        url_rec = urlparse(url_rec)
    if isinstance(u_rec, string_types):
        u_rec = urlparse(u_rec)
    return (url_rec.netloc == u_rec.netloc)
    # todo: collect more of sample cases.
    # disabled below check while working on ratholeradio, since links
    # could go to the parent and that is ok.  Figure out when it was
    # desired not to go to the parent -- we might need explicit option
    # and u_rec.path.startswith(url_rec.path)):


def dlurljoin(u_path, url):
    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO
    if url_rec.scheme:
        # independent full url, so just return it
        return url
    if u_path.endswith('/'):  # should here be also a scheme use?
        if url.startswith('/'): # jump to the root
            u_path_rec = urlparse(u_path)
            return urljoin(urlunparse((u_path_rec.scheme, u_path_rec.netloc, '','','','')), url)
        else:
            return os.path.join(u_path, url)
    # TODO: recall where all this dirname came from and bring into the test
    return urljoin(os.path.dirname(u_path) + '/', url)


# TODO should it be a node maybe?
class SimpleURLStamper(object):
    """Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header
    """
    def __init__(self, mode='full'):
        self.mode = mode

    def __call__(self, url):
        # Extracted from above madness
        # TODO: add mode alike to 'relaxed' where we would not
        # care about content-disposition filename
        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way
        request = Request(url)

        # No traffic compression since we do not know how to identify
        # exactly either it has to be decompressed
        # request.add_header('Accept-encoding', 'gzip,deflate')
        #
        # TODO: think about stamping etc -- we seems to be redoing
        # what git-annex does for us already... not really
        r = retry_urlopen(request)
        try:
            r_info = r.info()
            r_stamp = get_url_response_stamp(url, r_info)

            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)
        finally:
            r.close()


# TODO:  make it consistent/clear at what stage % encoding/decoding happens!
# now it is a mix!

#
# Useful functionality in requests.models
#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance
#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...
#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc
#

#@auto_repr
# TODO: Well -- it is more of a URI than URL I guess
class URL(object):
    """A helper class to deal with URLs with some "magical" treats to facilitate use of "ssh" urls

    Intended to be a R/O object (i.e. no fields should be changed in-place)

    Although largely decorating urlparse.ParseResult, it
    - doesn't mandate providing all parts of the URL
    - doesn't require netloc but rather asks for separate username, password, and hostname
    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements
      (or otherwise encoded), with empty value being equivalent with no value.  If conversion
      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id
      within a page)

    The idea is that this class should help to break apart a URL, while being
    able to rebuild itself into a string representation for reuse

    If scheme was implicit (e.g. "host:path" for ssh, or later may be "host::path" for
    rsync) ":implicit" suffix added to the scheme (so it is included in the repr of the
    instance "for free")

    Additional semantics we might want to somehow understand/map???
    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,
      with `location` (if not empty) identifying the remote (super)dataset

    Implementation:
    - currently might be overengineered and could be simplified.  Internally keeps
      all values as attributes, but for manipulation in dict (called fields) and converts
      back and forth to ParseResults
    So after we agree on viability of the approach  and collect enough tests we can simplify
    """

    # fields with their defaults
    _FIELDS = (
        'scheme',
        'username',
        'password',
        'hostname', 'port',
        'path',  # 'params',
        'query',
        'fragment',
    )

    __slots__ = _FIELDS + ('_fields', '_str')

    def __init__(self, url=None, **kwargs):
        if url and (bool(url) == bool(kwargs)):
            raise ValueError(
                "Specify either url or breakdown from the fields, not both. "
                "Got url=%r, fields=%r" % (url, kwargs))

        # ok -- let's default to all of them being an empty string
        # Originally used None to signal no value but it complicated
        # operations since often needed first a check before e.g. doing
        # .startswith.  So more harmonic is just to store strings.
        # Not provided seems to be exactly as empty string, so Ok (stdlib
        # guys knew what they were doing ;)
        self._fields = {f: '' for f in self._FIELDS}
        if url:
            self._set_from_str(url)
        else:
            self._set_from_fields(**kwargs)

    @property
    def is_implicit(self):
        return self._fields['scheme'].endswith(':implicit')

    @property
    def fields(self):
        """Returns shallow copy of fields to ease manipulations"""
        return self._fields.copy()

    def __repr__(self):
        # since auto_repr doesn't support "non-0" values atm
        return "%s(%s)" % (
            self.__class__.__name__,
            ", ".join(["%s=%r" % (k, v)
                       for k, v in sorted(self._fields.items())
                       if v]))

    # Some custom __str__s for :implicit URLs
    def __str_ssh__(self):
        """Custom str for ssh:implicit"""
        url = urlunparse(self.to_pr())
        pref = 'ssh:implicit://'
        assert(url.startswith(pref))
        url = url[len(pref):]
        # and we should replace leading /
        url = url.replace('/',
                          ':/' if self.path.startswith('/') else ':',
                          1)
        return url

    def __str_datalad__(self):
        """Custom str for datalad:implicit"""
        fields = self._fields.copy()
        fields['scheme'] = ''
        url = urlunparse(self._fields_to_pr(fields))
        if not fields['hostname']:
            # was of /// type
            url = '//' + url
        return url

    def __str_file__(self):
        """Custom str for datalad:implicit"""
        fields = self._fields.copy()
        fields['scheme'] = ''
        url = urlunparse(self._fields_to_pr(fields))
        return url

    # Lazily evaluated if _str was not set
    def __str__(self):
        if self._str is None:
            self._str = self._as_str()
        return self._str

    def _as_str(self):
        """Re-evaluate string repsentation"""

        if not self.is_implicit:
            return urlunparse(self.to_pr())
        else:
            base_scheme = self.scheme.split(':', 1)[0]
            try:
                __str__ = getattr(self, '__str_%s__' % base_scheme)
            except AttributeError:
                raise ValueError("Don't know how to convert %s:implicit into str"
                                 % base_scheme)
            return __str__()

    #
    # If any field is specified, URL is not considered 'False', i.e.
    # non-existing, although may be we could/shout omit having only
    # scheme or port specified since it doesn't point to any useful
    # location
    #

    def __nonzero__(self):
        fields = self._fields
        return any(fields.values())

    # for PY3
    __bool__ = __nonzero__

    #
    # Helpers to deal with internal structures and conversions
    #

    def _set_from_fields(self, **fields):
        unknown_fields = set(fields).difference(self._FIELDS)
        if unknown_fields:
            raise ValueError("Do not know about %s. Known fields are: %s"
                             % (unknown_fields, self._FIELDS))

        # encode dicts for query or fragment into
        for f in {'query', 'fragment'}:
            v = fields.get(f)
            if isinstance(v, dict):

                ev = urlencode(v)
                # / is reserved char within query
                if f == 'fragment' and '%2F' not in str(v):
                    # but seems to be ok'ish within the fragment which is
                    # the last element of URI and anyways used only by the
                    # client (i.e. by us here if used to compose the URL)
                    # so let's return / back for clarity if there were no
                    # awkward %2F to startwith
                    ev = ev.replace('%2F', '/')
                fields[f] = ev

        self._fields.update(fields)
        self._str = None

    def to_pr(self):
        return self._fields_to_pr(self._fields)

    @classmethod
    def _fields_to_pr(cls, fields):
        """Recompose back fields dict to ParseResult"""
        netloc = fields['username'] or ''
        if fields['password']:
            netloc += ':' + fields['password']
        if netloc:
            netloc += '@'
        netloc += fields['hostname']
        if fields['port']:
            netloc += ':%s' % fields['port']

        pr_fields = {
            f: fields[f]
            for f in cls._FIELDS
            if f not in ('hostname', 'password', 'username', 'port')
        }
        pr_fields['netloc'] = netloc
        pr_fields['params'] = ''

        return ParseResult(**pr_fields)

    def _pr_to_fields(self, pr):
        """ParseResult is a tuple so immutable, which complicates adjusting it

        This function converts ParseResult into dict"""

        if pr.params:
            lgr.warning("ParseResults contains params %r, which will be ignored"
                        % (pr.params,))

        # can't use just pr._asdict since we care to ask those properties
        # such as .port , .hostname etc
        # Forcing '' instead of None since those properties (.hostname), .password,
        # .username return None if not available and we decided to uniformize
        return {f: (getattr(pr, f) or '') for f in self._FIELDS}

    def _set_from_str(self, url):
        fields = self._pr_to_fields(urlparse(url))
        lgr.log(5, "Parsed url %s into fields %s" % (url, fields))

        # Special treatments
        # file:///path should stay file:
        if fields['scheme'] and fields['scheme'] not in {'file'} \
                and not fields['hostname']:
            # dl+archive:... or just for ssh   hostname:path/p1
            if '+' not in fields['scheme']:
                fields['hostname'] = fields['scheme']
                fields['scheme'] = 'ssh:implicit'
                lgr.log(5, "Assuming ssh style url, adjusted: %s" % (fields,))

        if not fields['scheme'] and not fields['hostname']:
            if fields['path'] and '@' in fields['path']:
                # user@host:path/sp1
                fields['scheme'] = 'ssh:implicit'
            elif url.startswith('//'):
                # e.g. // or ///path
                fields['scheme'] = 'datalad:implicit'
            else:
                fields['scheme'] = 'file:implicit'

        if not fields['scheme'] and fields['hostname']:
            # e.g. //a/path
            fields['scheme'] = 'datalad:implicit'

        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:
            if fields['query'] or fields['fragment']:
                # actually might be legit with some obscure filenames,
                # so TODO for correct -- probably just re.match the entire url
                # into fields
                raise ValueError("Thought that this url would point to host:path but got query and fragments: %s"
                                 % str(fields))
            #  user@hostname[:path] --> ssh://user@hostname[/path]
            parts = _split_colon(url)
            fields['scheme'] = 'ssh:implicit'
            if len(parts) == 2:
                fields['hostname'] = parts[0]
                fields['path'] = parts[1]
            # hostname might still contain the password
            if fields['hostname'] and '@' in fields['hostname']:
                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)
            lgr.log(5, "Detected ssh style url, adjusted: %s" % (fields,))

        self._set_from_fields(**fields)
        self._str = url

        # well -- some urls might not unparse identically back
        # strictly speaking, but let's assume they do
        url_ = self._as_str()
        if url != url_:
            lgr.warning("Parsed version of url %r differs from original %r",
                        url_, url)

    #
    # Quick comparators
    #

    def __eq__(self, other):
        if not isinstance(other, URL):
            other = URL(other)
        return other._fields == self._fields

    def __ne__(self, other):
        return not (self == other)

    #
    # Access helpers
    #

    def _parse_qs(self, s, auto_delist=True):
        """Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values"""
        if not s:
            return {}
        out = OrderedDict(parse_qsl(s, 1))
        if not auto_delist:
            return out
        for k in out:
            v = out[k]
            if isinstance(v, list) and len(v) == 1:
                v = v[0]
                out[k] = None if v == '' else v
        return out

    @property
    def query_dict(self):
        return self._parse_qs(self.query)

    @property
    def fragment_dict(self):
        return self._parse_qs(self.fragment)

    # def __getattribute__(self, item):
    #     if item.startswith('_') or item not in URL._FIELDS:
    #         return super(URL, self).__getattribute__(item)
    #     else:
    #         return self._fields[item]

# Bind properties to access fields (without overriding __getattr*)
# This one doesn't work -- I guess due to absent binding of f value
# to the context
#for f in URL._FIELDS:
#   setattr(URL, f, property(lambda self: self._fields[f]))
# These work but ugly duplication
# URL.hostname = property(lambda self: self._fields['hostname'])
# URL.path = property(lambda self: self._fields['path'])
# URL.query = property(lambda self: self._fields['query'])
# URL.scheme = property(lambda self: self._fields['scheme'])
# URL.fragment = property(lambda self: self._fields['fragment'])
# URL.username = property(lambda self: self._fields['username'])
# URL.password = property(lambda self: self._fields['password'])
# URL.port = property(lambda self: self._fields['port'])
for f in URL._FIELDS:
    exec("URL.%s = property(lambda self: self._fields['%s'])" % (f, f))


def _split_colon(s, maxsplit=1):
    """Split on unescaped colon"""
    return re.compile(r'(?<!\\):').split(s, maxsplit=maxsplit)


def parse_url_opts(url):
    """Given a string with url-style query, split into content before # and options as dict"""
    url = URL(url)
    # we need to filter out query and fragment to get the base url
    fields = url.fields
    fields.pop('query')
    fields.pop('fragment')
    opts = url.query_dict
    return str(URL(**fields)), opts


# TODO: should we just define URL.good_for_git or smth like that? ;)
# although git also understands regular paths
def is_url(s):
    """Returns whether a string looks like something what datalad should treat as a URL

    This includes ssh "urls" which git understands.
    """
    try:
        url = URL(s)
    except:
        return False
    implicit = url.is_implicit
    scheme = url.scheme
    return scheme in {'ssh:implicit'} or (not implicit and bool(url))


#### windows workaround ###
# TODO: There should be a better way
def get_local_file_url(fname):
    """Return OS specific URL pointing to a local file

    Parameters
    ----------
    fname : string
        Filename.  If not absolute, abspath is used
    """
    fname = fname if isabs(fname) else abspath(fname)
    if on_windows:
        fname_rep = fname.replace('\\', '/')
        furl = "file:///%s" % urlquote(fname_rep)
        lgr.debug("Replaced '\\' in file\'s url: %s" % furl)
    else:
        # TODO:  need to fix for all the encoding etc
        furl = str(URL(scheme='file', path=urlquote(fname)))
    return furl


def get_local_path_from_url(url):
    """If given a file:// URL, returns a local path, if possible.

    Raises `ValueError` if not possible, for example, if the URL
    scheme is different, or if the `host` isn't empty or 'localhost'

    The returned path is always absolute.
    """
    urlparts = urlsplit(url)
    if not urlparts.scheme == 'file':
        raise ValueError(
            "Non 'file://' URL cannot be resolved to a local path")
    if not (urlparts.netloc in ('', 'localhost', '::1') \
            or urlparts.netloc.startswith('127.')):
        raise ValueError("file:// URL does not point to 'localhost'")
    return urlunquote(urlparts.path)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Incorrect Parsing of SSH Implicit URLs Causes Parsing Failures

**Description:**
When parsing SSH implicit URLs that include a hostname followed by a colon and a path, the URL is incorrectly interpreted using the `file:implicit` scheme instead of `ssh:implicit`. This leads to incorrect separation of the hostname and path components, causing downstream operations to fail.

**Example Code:**
```python
from datalad.support.network import URL

url = 'weired_url:/'
parsed_url = URL(url)

print(parsed_url.scheme)      # Expected: 'ssh:implicit'
print(parsed_url.hostname)   # Expected: 'weired_url'
print(parsed_url.path)       # Expected: '/'
```

**Expected Behavior:**
The URL should be parsed with the `ssh:implicit` scheme, correctly identifying the hostname as `'weired_url'` and the path as `'/'`.

**Actual Behavior:**
The URL is parsed with the `file:implicit` scheme, resulting in the entire string `'weired_url:/'` being set as the path and leaving the hostname undefined. This mismatch causes assertions and operations that rely on proper URL parsing to fail.

[/ISSUE]
{
    "test_same_website": "PASSED",
    "test_get_tld": "PASSED",
    "test_dlurljoin": "PASSED",
    "test_rfc2822_to_epoch": "PASSED",
    "test_get_response_disposition_filename": "PASSED",
    "test_parse_url_opts": "PASSED",
    "test_split_colon": "PASSED",
    "test_url_eq": "PASSED",
    "test_url_base": "PASSED",
    "test_url_samples": "PASSED",
    "test_url_compose_archive_one": "PASSED",
    "test_url_fragments_and_query": "PASSED",
    "test_url_dicts": "PASSED",
    "test_get_url_path_on_fileurls": "PASSED",
    "test_is_url": "PASSED",
    "test_get_local_path_from_url": "PASSED",
    "test_get_local_file_url_linux": "FAILED"
}
diff --git a/datalad/support/network.py b/datalad/support/network.py
index 88fbba7e0..c5510d6c9 100644
--- a/datalad/support/network.py
+++ b/datalad/support/network.py
@@ -348,10 +348,16 @@ class URL(object):
         pref = 'ssh:implicit://'
         assert(url.startswith(pref))
         url = url[len(pref):]
-        # and we should replace leading /
-        url = url.replace('/',
-                          ':/' if self.path.startswith('/') else ':',
-                          1)
+        # and we should replace leading / after the hostname
+        # Will preserve hostname as is, since we are trying to be as silly as ssh
+        # where it tries to resolve hostnames with /
+        lhostname = len(self.hostname)
+
+        url = url[:lhostname] + \
+              url[lhostname:].replace(
+                  '/',
+                  ':/' if self.path.startswith('/') else ':',
+                  1)
         return url
 
     def __str_datalad__(self):
@@ -477,7 +483,6 @@ class URL(object):
     def _set_from_str(self, url):
         fields = self._pr_to_fields(urlparse(url))
         lgr.log(5, "Parsed url %s into fields %s" % (url, fields))
-
         # Special treatments
         # file:///path should stay file:
         if fields['scheme'] and fields['scheme'] not in {'file'} \
@@ -489,8 +494,11 @@ class URL(object):
                 lgr.log(5, "Assuming ssh style url, adjusted: %s" % (fields,))
 
         if not fields['scheme'] and not fields['hostname']:
-            if fields['path'] and '@' in fields['path']:
+            parts = _split_colon(url)
+            if fields['path'] and '@' in fields['path'] or len(parts) > 1:
                 # user@host:path/sp1
+                # or host_name: (hence parts check)
+                # TODO: we need a regex to catch those really, parts check is not suff
                 fields['scheme'] = 'ssh:implicit'
             elif url.startswith('//'):
                 # e.g. // or ///path

{"datalad/support/network.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\nimport calendar\nimport email.utils\nimport gzip\nimport os\nimport re\nimport shutil\nimport time\n\nfrom collections import OrderedDict\nfrom os.path import abspath, isabs\n\nfrom six import string_types\nfrom six import iteritems\nfrom six.moves.urllib.request import urlopen, Request\nfrom six.moves.urllib.parse import quote as urlquote, unquote as urlunquote\nfrom six.moves.urllib.parse import urljoin, urlparse, urlsplit, urlunsplit, urlunparse, ParseResult\nfrom six.moves.urllib.parse import parse_qsl\nfrom six.moves.urllib.parse import urlencode\nfrom six.moves.urllib.error import URLError\n\nfrom datalad.utils import on_windows\n\nfrom datalad.utils import auto_repr\n# TODO not sure what needs to use `six` here yet\nimport requests\n\nimport logging\nlgr = logging.getLogger('datalad.network')\n\n\ndef get_response_disposition_filename(s):\n    \"\"\"Given a string s as from HTTP Content-Disposition field in the response\n    return possibly present filename if any\n    \"\"\"\n    if not s:\n        return None\n    # If the response has Content-Disposition, try to get filename from it\n    cd = dict(map(\n        lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),\n        s.split(';')))\n    if 'filename' in cd:\n        filename = cd['filename'].strip(\"\\\"'\")\n        return filename\n    return None\n\n\ndef get_url_disposition_filename(url, headers=None):\n    \"\"\"Get filename as possibly provided by the server in Content-Disposition\n    \"\"\"\n    if headers is None:\n        request = Request(url)\n        r = retry_urlopen(request)\n        # things are different in requests\n        if 'requests.' in str(r.__class__):\n            headers = r.headers\n        else:\n            headers = r.info()\n    else:\n        r = None\n    try:\n        return get_response_disposition_filename(headers.get('Content-Disposition', ''))\n    finally:\n        if r:\n            r.close()\n\n\ndef get_url_straight_filename(url, strip=[], allowdir=False):\n    \"\"\"Get file/dir name of the last path component of the URL\n\n    Parameters\n    ----------\n    strip: list, optional\n      If provided, listed names will not be considered and their\n      parent directory will be selected\n    allowdir: bool, optional\n      If url points to a \"directory\" (ends with /), empty string\n      would be returned unless allowdir is True, in which case the\n      name of the directory would be returned\n    \"\"\"\n    path = urlunquote(urlsplit(url).path)\n    path_parts = path.split('/')\n\n    if allowdir:\n        # strip empty ones\n        while len(path_parts) > 1 and not path_parts[-1]:\n            path_parts = path_parts[:-1]\n\n    if strip:\n        while path_parts and path_parts[-1] in strip:\n            path_parts = path_parts[:-1]\n\n    if path_parts:\n        return path_parts[-1]\n    else:\n        return None\n\n\ndef get_url_filename(url, headers=None, strip=[]):\n    \"\"\"Get filename from the url, first consulting server about Content-Disposition\n    \"\"\"\n    filename = get_url_disposition_filename(url, headers)\n    if filename:\n        return filename\n    return get_url_straight_filename(url, strip=[])\n\n\ndef get_url_response_stamp(url, response_info):\n    size, mtime = None, None\n    if 'Content-length' in response_info:\n        size = int(response_info['Content-length'])\n    if 'Last-modified' in response_info:\n        mtime = calendar.timegm(email.utils.parsedate(\n            response_info['Last-modified']))\n    return dict(size=size, mtime=mtime, url=url)\n\n\ndef get_tld(url):\n    \"\"\"Return top level domain from a url\n\n    Parameters\n    ----------\n    url : str\n    \"\"\"\n    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld\n    if not url.strip():\n        raise ValueError(\"Empty URL has no TLD\")\n    rec = urlsplit(url)\n    if not rec.netloc:\n        if not rec.scheme:\n            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'\n            return url.split('/', 1)[0]\n        else:\n            raise ValueError(\"It seems that only the scheme was provided without the net location/TLD\")\n    return rec.netloc\n\n\nfrom email.utils import parsedate_tz, mktime_tz\ndef rfc2822_to_epoch(datestr):\n    \"\"\"Given rfc2822 date/time format, return seconds since epoch\"\"\"\n    return mktime_tz(parsedate_tz(datestr))\n\n\nimport calendar\nfrom datetime import datetime\ndef iso8601_to_epoch(datestr):\n    return calendar.timegm(datetime.strptime(datestr, \"%Y-%m-%dT%H:%M:%S.%fZ\").timetuple())\n\n\ndef __urlopen_requests(url, header_vals=None):\n    # XXX Workaround for now for ... broken code\n    if isinstance(url, Request):\n        url = url.get_full_url()\n    return requests.Session().get(url)\n\n\ndef retry_urlopen(url, retries=3):\n    for t in range(retries):\n        try:\n            return __urlopen_requests(url)\n        except URLError as e:\n            lgr.warn(\"Received exception while reading %s: %s\" % (url, e))\n            if t == retries - 1:\n                # if we have reached allowed number of retries -- reraise\n                raise\n\n\ndef is_url_quoted(url):\n    \"\"\"Return either URL looks being already quoted\n    \"\"\"\n    try:\n        url_ = urlunquote(url)\n        return url != url_\n    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)\n        return False\n\n\ndef same_website(url_rec, u_rec):\n    \"\"\"Decide either a link leads to external site\n\n    Parameters\n    ----------\n    url_rec: ParseResult\n      record for original url\n    u_rec: ParseResult\n      record for new url\n    \"\"\"\n    if isinstance(url_rec, string_types):\n        url_rec = urlparse(url_rec)\n    if isinstance(u_rec, string_types):\n        u_rec = urlparse(u_rec)\n    return (url_rec.netloc == u_rec.netloc)\n    # todo: collect more of sample cases.\n    # disabled below check while working on ratholeradio, since links\n    # could go to the parent and that is ok.  Figure out when it was\n    # desired not to go to the parent -- we might need explicit option\n    # and u_rec.path.startswith(url_rec.path)):\n\n\ndef dlurljoin(u_path, url):\n    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO\n    if url_rec.scheme:\n        # independent full url, so just return it\n        return url\n    if u_path.endswith('/'):  # should here be also a scheme use?\n        if url.startswith('/'): # jump to the root\n            u_path_rec = urlparse(u_path)\n            return urljoin(urlunparse((u_path_rec.scheme, u_path_rec.netloc, '','','','')), url)\n        else:\n            return os.path.join(u_path, url)\n    # TODO: recall where all this dirname came from and bring into the test\n    return urljoin(os.path.dirname(u_path) + '/', url)\n\n\n# TODO should it be a node maybe?\nclass SimpleURLStamper(object):\n    \"\"\"Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header\n    \"\"\"\n    def __init__(self, mode='full'):\n        self.mode = mode\n\n    def __call__(self, url):\n        # Extracted from above madness\n        # TODO: add mode alike to 'relaxed' where we would not\n        # care about content-disposition filename\n        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way\n        request = Request(url)\n\n        # No traffic compression since we do not know how to identify\n        # exactly either it has to be decompressed\n        # request.add_header('Accept-encoding', 'gzip,deflate')\n        #\n        # TODO: think about stamping etc -- we seems to be redoing\n        # what git-annex does for us already... not really\n        r = retry_urlopen(request)\n        try:\n            r_info = r.info()\n            r_stamp = get_url_response_stamp(url, r_info)\n\n            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)\n        finally:\n            r.close()\n\n\n# TODO:  make it consistent/clear at what stage % encoding/decoding happens!\n# now it is a mix!\n\n#\n# Useful functionality in requests.models\n#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance\n#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...\n#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc\n#\n\n#@auto_repr\n# TODO: Well -- it is more of a URI than URL I guess\nclass URL(object):\n    \"\"\"A helper class to deal with URLs with some \"magical\" treats to facilitate use of \"ssh\" urls\n\n    Intended to be a R/O object (i.e. no fields should be changed in-place)\n\n    Although largely decorating urlparse.ParseResult, it\n    - doesn't mandate providing all parts of the URL\n    - doesn't require netloc but rather asks for separate username, password, and hostname\n    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements\n      (or otherwise encoded), with empty value being equivalent with no value.  If conversion\n      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id\n      within a page)\n\n    The idea is that this class should help to break apart a URL, while being\n    able to rebuild itself into a string representation for reuse\n\n    If scheme was implicit (e.g. \"host:path\" for ssh, or later may be \"host::path\" for\n    rsync) \":implicit\" suffix added to the scheme (so it is included in the repr of the\n    instance \"for free\")\n\n    Additional semantics we might want to somehow understand/map???\n    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,\n      with `location` (if not empty) identifying the remote (super)dataset\n\n    Implementation:\n    - currently might be overengineered and could be simplified.  Internally keeps\n      all values as attributes, but for manipulation in dict (called fields) and converts\n      back and forth to ParseResults\n    So after we agree on viability of the approach  and collect enough tests we can simplify\n    \"\"\"\n\n    # fields with their defaults\n    _FIELDS = (\n        'scheme',\n        'username',\n        'password',\n        'hostname', 'port',\n        'path',  # 'params',\n        'query',\n        'fragment',\n    )\n\n    __slots__ = _FIELDS + ('_fields', '_str')\n\n    def __init__(self, url=None, **kwargs):\n        if url and (bool(url) == bool(kwargs)):\n            raise ValueError(\n                \"Specify either url or breakdown from the fields, not both. \"\n                \"Got url=%r, fields=%r\" % (url, kwargs))\n\n        # ok -- let's default to all of them being an empty string\n        # Originally used None to signal no value but it complicated\n        # operations since often needed first a check before e.g. doing\n        # .startswith.  So more harmonic is just to store strings.\n        # Not provided seems to be exactly as empty string, so Ok (stdlib\n        # guys knew what they were doing ;)\n        self._fields = {f: '' for f in self._FIELDS}\n        if url:\n            self._set_from_str(url)\n        else:\n            self._set_from_fields(**kwargs)\n\n    @property\n    def is_implicit(self):\n        return self._fields['scheme'].endswith(':implicit')\n\n    @property\n    def fields(self):\n        \"\"\"Returns shallow copy of fields to ease manipulations\"\"\"\n        return self._fields.copy()\n\n    def __repr__(self):\n        # since auto_repr doesn't support \"non-0\" values atm\n        return \"%s(%s)\" % (\n            self.__class__.__name__,\n            \", \".join([\"%s=%r\" % (k, v)\n                       for k, v in sorted(self._fields.items())\n                       if v]))\n\n    # Some custom __str__s for :implicit URLs\n    def __str_ssh__(self):\n        \"\"\"Custom str for ssh:implicit\"\"\"\n        url = urlunparse(self.to_pr())\n        pref = 'ssh:implicit://'\n        assert(url.startswith(pref))\n        url = url[len(pref):]\n        # and we should replace leading /\n        url = url.replace('/',\n                          ':/' if self.path.startswith('/') else ':',\n                          1)\n        return url\n\n    def __str_datalad__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        if not fields['hostname']:\n            # was of /// type\n            url = '//' + url\n        return url\n\n    def __str_file__(self):\n        \"\"\"Custom str for datalad:implicit\"\"\"\n        fields = self._fields.copy()\n        fields['scheme'] = ''\n        url = urlunparse(self._fields_to_pr(fields))\n        return url\n\n    # Lazily evaluated if _str was not set\n    def __str__(self):\n        if self._str is None:\n            self._str = self._as_str()\n        return self._str\n\n    def _as_str(self):\n        \"\"\"Re-evaluate string repsentation\"\"\"\n\n        if not self.is_implicit:\n            return urlunparse(self.to_pr())\n        else:\n            base_scheme = self.scheme.split(':', 1)[0]\n            try:\n                __str__ = getattr(self, '__str_%s__' % base_scheme)\n            except AttributeError:\n                raise ValueError(\"Don't know how to convert %s:implicit into str\"\n                                 % base_scheme)\n            return __str__()\n\n    #\n    # If any field is specified, URL is not considered 'False', i.e.\n    # non-existing, although may be we could/shout omit having only\n    # scheme or port specified since it doesn't point to any useful\n    # location\n    #\n\n    def __nonzero__(self):\n        fields = self._fields\n        return any(fields.values())\n\n    # for PY3\n    __bool__ = __nonzero__\n\n    #\n    # Helpers to deal with internal structures and conversions\n    #\n\n    def _set_from_fields(self, **fields):\n        unknown_fields = set(fields).difference(self._FIELDS)\n        if unknown_fields:\n            raise ValueError(\"Do not know about %s. Known fields are: %s\"\n                             % (unknown_fields, self._FIELDS))\n\n        # encode dicts for query or fragment into\n        for f in {'query', 'fragment'}:\n            v = fields.get(f)\n            if isinstance(v, dict):\n\n                ev = urlencode(v)\n                # / is reserved char within query\n                if f == 'fragment' and '%2F' not in str(v):\n                    # but seems to be ok'ish within the fragment which is\n                    # the last element of URI and anyways used only by the\n                    # client (i.e. by us here if used to compose the URL)\n                    # so let's return / back for clarity if there were no\n                    # awkward %2F to startwith\n                    ev = ev.replace('%2F', '/')\n                fields[f] = ev\n\n        self._fields.update(fields)\n        self._str = None\n\n    def to_pr(self):\n        return self._fields_to_pr(self._fields)\n\n    @classmethod\n    def _fields_to_pr(cls, fields):\n        \"\"\"Recompose back fields dict to ParseResult\"\"\"\n        netloc = fields['username'] or ''\n        if fields['password']:\n            netloc += ':' + fields['password']\n        if netloc:\n            netloc += '@'\n        netloc += fields['hostname']\n        if fields['port']:\n            netloc += ':%s' % fields['port']\n\n        pr_fields = {\n            f: fields[f]\n            for f in cls._FIELDS\n            if f not in ('hostname', 'password', 'username', 'port')\n        }\n        pr_fields['netloc'] = netloc\n        pr_fields['params'] = ''\n\n        return ParseResult(**pr_fields)\n\n    def _pr_to_fields(self, pr):\n        \"\"\"ParseResult is a tuple so immutable, which complicates adjusting it\n\n        This function converts ParseResult into dict\"\"\"\n\n        if pr.params:\n            lgr.warning(\"ParseResults contains params %r, which will be ignored\"\n                        % (pr.params,))\n\n        # can't use just pr._asdict since we care to ask those properties\n        # such as .port , .hostname etc\n        # Forcing '' instead of None since those properties (.hostname), .password,\n        # .username return None if not available and we decided to uniformize\n        return {f: (getattr(pr, f) or '') for f in self._FIELDS}\n\n    def _set_from_str(self, url):\n        fields = self._pr_to_fields(urlparse(url))\n        lgr.log(5, \"Parsed url %s into fields %s\" % (url, fields))\n\n        # Special treatments\n        # file:///path should stay file:\n        if fields['scheme'] and fields['scheme'] not in {'file'} \\\n                and not fields['hostname']:\n            # dl+archive:... or just for ssh   hostname:path/p1\n            if '+' not in fields['scheme']:\n                fields['hostname'] = fields['scheme']\n                fields['scheme'] = 'ssh:implicit'\n                lgr.log(5, \"Assuming ssh style url, adjusted: %s\" % (fields,))\n\n        if not fields['scheme'] and not fields['hostname']:\n            if fields['path'] and '@' in fields['path']:\n                # user@host:path/sp1\n                fields['scheme'] = 'ssh:implicit'\n            elif url.startswith('//'):\n                # e.g. // or ///path\n                fields['scheme'] = 'datalad:implicit'\n            else:\n                fields['scheme'] = 'file:implicit'\n\n        if not fields['scheme'] and fields['hostname']:\n            # e.g. //a/path\n            fields['scheme'] = 'datalad:implicit'\n\n        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:\n            if fields['query'] or fields['fragment']:\n                # actually might be legit with some obscure filenames,\n                # so TODO for correct -- probably just re.match the entire url\n                # into fields\n                raise ValueError(\"Thought that this url would point to host:path but got query and fragments: %s\"\n                                 % str(fields))\n            #  user@hostname[:path] --> ssh://user@hostname[/path]\n            parts = _split_colon(url)\n            fields['scheme'] = 'ssh:implicit'\n            if len(parts) == 2:\n                fields['hostname'] = parts[0]\n                fields['path'] = parts[1]\n            # hostname might still contain the password\n            if fields['hostname'] and '@' in fields['hostname']:\n                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)\n            lgr.log(5, \"Detected ssh style url, adjusted: %s\" % (fields,))\n\n        self._set_from_fields(**fields)\n        self._str = url\n\n        # well -- some urls might not unparse identically back\n        # strictly speaking, but let's assume they do\n        url_ = self._as_str()\n        if url != url_:\n            lgr.warning(\"Parsed version of url %r differs from original %r\",\n                        url_, url)\n\n    #\n    # Quick comparators\n    #\n\n    def __eq__(self, other):\n        if not isinstance(other, URL):\n            other = URL(other)\n        return other._fields == self._fields\n\n    def __ne__(self, other):\n        return not (self == other)\n\n    #\n    # Access helpers\n    #\n\n    def _parse_qs(self, s, auto_delist=True):\n        \"\"\"Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values\"\"\"\n        if not s:\n            return {}\n        out = OrderedDict(parse_qsl(s, 1))\n        if not auto_delist:\n            return out\n        for k in out:\n            v = out[k]\n            if isinstance(v, list) and len(v) == 1:\n                v = v[0]\n                out[k] = None if v == '' else v\n        return out\n\n    @property\n    def query_dict(self):\n        return self._parse_qs(self.query)\n\n    @property\n    def fragment_dict(self):\n        return self._parse_qs(self.fragment)\n\n    # def __getattribute__(self, item):\n    #     if item.startswith('_') or item not in URL._FIELDS:\n    #         return super(URL, self).__getattribute__(item)\n    #     else:\n    #         return self._fields[item]\n\n# Bind properties to access fields (without overriding __getattr*)\n# This one doesn't work -- I guess due to absent binding of f value\n# to the context\n#for f in URL._FIELDS:\n#   setattr(URL, f, property(lambda self: self._fields[f]))\n# These work but ugly duplication\n# URL.hostname = property(lambda self: self._fields['hostname'])\n# URL.path = property(lambda self: self._fields['path'])\n# URL.query = property(lambda self: self._fields['query'])\n# URL.scheme = property(lambda self: self._fields['scheme'])\n# URL.fragment = property(lambda self: self._fields['fragment'])\n# URL.username = property(lambda self: self._fields['username'])\n# URL.password = property(lambda self: self._fields['password'])\n# URL.port = property(lambda self: self._fields['port'])\nfor f in URL._FIELDS:\n    exec(\"URL.%s = property(lambda self: self._fields['%s'])\" % (f, f))\n\n\ndef _split_colon(s, maxsplit=1):\n    \"\"\"Split on unescaped colon\"\"\"\n    return re.compile(r'(?<!\\\\):').split(s, maxsplit=maxsplit)\n\n\ndef parse_url_opts(url):\n    \"\"\"Given a string with url-style query, split into content before # and options as dict\"\"\"\n    url = URL(url)\n    # we need to filter out query and fragment to get the base url\n    fields = url.fields\n    fields.pop('query')\n    fields.pop('fragment')\n    opts = url.query_dict\n    return str(URL(**fields)), opts\n\n\n# TODO: should we just define URL.good_for_git or smth like that? ;)\n# although git also understands regular paths\ndef is_url(s):\n    \"\"\"Returns whether a string looks like something what datalad should treat as a URL\n\n    This includes ssh \"urls\" which git understands.\n    \"\"\"\n    try:\n        url = URL(s)\n    except:\n        return False\n    implicit = url.is_implicit\n    scheme = url.scheme\n    return scheme in {'ssh:implicit'} or (not implicit and bool(url))\n\n\n#### windows workaround ###\n# TODO: There should be a better way\ndef get_local_file_url(fname):\n    \"\"\"Return OS specific URL pointing to a local file\n\n    Parameters\n    ----------\n    fname : string\n        Filename.  If not absolute, abspath is used\n    \"\"\"\n    fname = fname if isabs(fname) else abspath(fname)\n    if on_windows:\n        fname_rep = fname.replace('\\\\', '/')\n        furl = \"file:///%s\" % urlquote(fname_rep)\n        lgr.debug(\"Replaced '\\\\' in file\\'s url: %s\" % furl)\n    else:\n        # TODO:  need to fix for all the encoding etc\n        furl = str(URL(scheme='file', path=urlquote(fname)))\n    return furl\n\n\ndef get_local_path_from_url(url):\n    \"\"\"If given a file:// URL, returns a local path, if possible.\n\n    Raises `ValueError` if not possible, for example, if the URL\n    scheme is different, or if the `host` isn't empty or 'localhost'\n\n    The returned path is always absolute.\n    \"\"\"\n    urlparts = urlsplit(url)\n    if not urlparts.scheme == 'file':\n        raise ValueError(\n            \"Non 'file://' URL cannot be resolved to a local path\")\n    if not (urlparts.netloc in ('', 'localhost', '::1') \\\n            or urlparts.netloc.startswith('127.')):\n        raise ValueError(\"file:// URL does not point to 'localhost'\")\n    return urlunquote(urlparts.path)\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Incorrect Parsing of SSH Implicit URLs Causes Parsing Failures

**Description:**
When parsing SSH implicit URLs that include a hostname followed by a colon and a path, the URL is incorrectly interpreted using the `file:implicit` scheme instead of `ssh:implicit`. This leads to incorrect separation of the hostname and path components, causing downstream operations to fail.

**Example Code:**
```python
from datalad.support.network import URL

url = 'weired_url:/'
parsed_url = URL(url)

print(parsed_url.scheme)      # Expected: 'ssh:implicit'
print(parsed_url.hostname)   # Expected: 'weired_url'
print(parsed_url.path)       # Expected: '/'
```

**Expected Behavior:**
The URL should be parsed with the `ssh:implicit` scheme, correctly identifying the hostname as `'weired_url'` and the path as `'/'`.

**Actual Behavior:**
The URL is parsed with the `file:implicit` scheme, resulting in the entire string `'weired_url:/'` being set as the path and leaving the hostname undefined. This mismatch causes assertions and operations that rely on proper URL parsing to fail.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/network.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

import calendar
import email.utils
import gzip
import os
import re
import shutil
import time

from collections import OrderedDict
from os.path import abspath, isabs

from six import string_types
from six import iteritems
from six.moves.urllib.request import urlopen, Request
from six.moves.urllib.parse import quote as urlquote, unquote as urlunquote
from six.moves.urllib.parse import urljoin, urlparse, urlsplit, urlunsplit, urlunparse, ParseResult
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlencode
from six.moves.urllib.error import URLError

from datalad.utils import on_windows

from datalad.utils import auto_repr
# TODO not sure what needs to use `six` here yet
import requests

import logging
lgr = logging.getLogger('datalad.network')


def get_response_disposition_filename(s):
    """Given a string s as from HTTP Content-Disposition field in the response
    return possibly present filename if any
    """
    if not s:
        return None
    # If the response has Content-Disposition, try to get filename from it
    cd = dict(map(
        lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),
        s.split(';')))
    if 'filename' in cd:
        filename = cd['filename'].strip("\"'")
        return filename
    return None


def get_url_disposition_filename(url, headers=None):
    """Get filename as possibly provided by the server in Content-Disposition
    """
    if headers is None:
        request = Request(url)
        r = retry_urlopen(request)
        # things are different in requests
        if 'requests.' in str(r.__class__):
            headers = r.headers
        else:
            headers = r.info()
    else:
        r = None
    try:
        return get_response_disposition_filename(headers.get('Content-Disposition', ''))
    finally:
        if r:
            r.close()


def get_url_straight_filename(url, strip=[], allowdir=False):
    """Get file/dir name of the last path component of the URL

    Parameters
    ----------
    strip: list, optional
      If provided, listed names will not be considered and their
      parent directory will be selected
    allowdir: bool, optional
      If url points to a "directory" (ends with /), empty string
      would be returned unless allowdir is True, in which case the
      name of the directory would be returned
    """
    path = urlunquote(urlsplit(url).path)
    path_parts = path.split('/')

    if allowdir:
        # strip empty ones
        while len(path_parts) > 1 and not path_parts[-1]:
            path_parts = path_parts[:-1]

    if strip:
        while path_parts and path_parts[-1] in strip:
            path_parts = path_parts[:-1]

    if path_parts:
        return path_parts[-1]
    else:
        return None


def get_url_filename(url, headers=None, strip=[]):
    """Get filename from the url, first consulting server about Content-Disposition
    """
    filename = get_url_disposition_filename(url, headers)
    if filename:
        return filename
    return get_url_straight_filename(url, strip=[])


def get_url_response_stamp(url, response_info):
    size, mtime = None, None
    if 'Content-length' in response_info:
        size = int(response_info['Content-length'])
    if 'Last-modified' in response_info:
        mtime = calendar.timegm(email.utils.parsedate(
            response_info['Last-modified']))
    return dict(size=size, mtime=mtime, url=url)


def get_tld(url):
    """Return top level domain from a url

    Parameters
    ----------
    url : str
    """
    # maybe use this instead to be safe:  https://pypi.python.org/pypi/tld
    if not url.strip():
        raise ValueError("Empty URL has no TLD")
    rec = urlsplit(url)
    if not rec.netloc:
        if not rec.scheme:
            # There were no scheme provided thus netloc was empty -- must have been a simple 'path like'
            return url.split('/', 1)[0]
        else:
            raise ValueError("It seems that only the scheme was provided without the net location/TLD")
    return rec.netloc


from email.utils import parsedate_tz, mktime_tz
def rfc2822_to_epoch(datestr):
    """Given rfc2822 date/time format, return seconds since epoch"""
    return mktime_tz(parsedate_tz(datestr))


import calendar
from datetime import datetime
def iso8601_to_epoch(datestr):
    return calendar.timegm(datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%S.%fZ").timetuple())


def __urlopen_requests(url, header_vals=None):
    # XXX Workaround for now for ... broken code
    if isinstance(url, Request):
        url = url.get_full_url()
    return requests.Session().get(url)


def retry_urlopen(url, retries=3):
    for t in range(retries):
        try:
            return __urlopen_requests(url)
        except URLError as e:
            lgr.warn("Received exception while reading %s: %s" % (url, e))
            if t == retries - 1:
                # if we have reached allowed number of retries -- reraise
                raise


def is_url_quoted(url):
    """Return either URL looks being already quoted
    """
    try:
        url_ = urlunquote(url)
        return url != url_
    except:  # problem with unquoting -- then it must be wasn't quoted (correctly)
        return False


def same_website(url_rec, u_rec):
    """Decide either a link leads to external site

    Parameters
    ----------
    url_rec: ParseResult
      record for original url
    u_rec: ParseResult
      record for new url
    """
    if isinstance(url_rec, string_types):
        url_rec = urlparse(url_rec)
    if isinstance(u_rec, string_types):
        u_rec = urlparse(u_rec)
    return (url_rec.netloc == u_rec.netloc)
    # todo: collect more of sample cases.
    # disabled below check while working on ratholeradio, since links
    # could go to the parent and that is ok.  Figure out when it was
    # desired not to go to the parent -- we might need explicit option
    # and u_rec.path.startswith(url_rec.path)):


def dlurljoin(u_path, url):
    url_rec = urlparse(url)  # probably duplicating parsing :-/ TODO
    if url_rec.scheme:
        # independent full url, so just return it
        return url
    if u_path.endswith('/'):  # should here be also a scheme use?
        if url.startswith('/'): # jump to the root
            u_path_rec = urlparse(u_path)
            return urljoin(urlunparse((u_path_rec.scheme, u_path_rec.netloc, '','','','')), url)
        else:
            return os.path.join(u_path, url)
    # TODO: recall where all this dirname came from and bring into the test
    return urljoin(os.path.dirname(u_path) + '/', url)


# TODO should it be a node maybe?
class SimpleURLStamper(object):
    """Gets a simple stamp about the URL: {url, time, size} of whatever was provided in the header
    """
    def __init__(self, mode='full'):
        self.mode = mode

    def __call__(self, url):
        # Extracted from above madness
        # TODO: add mode alike to 'relaxed' where we would not
        # care about content-disposition filename
        # http://stackoverflow.com/questions/862173/how-to-download-a-file-using-python-in-a-smarter-way
        request = Request(url)

        # No traffic compression since we do not know how to identify
        # exactly either it has to be decompressed
        # request.add_header('Accept-encoding', 'gzip,deflate')
        #
        # TODO: think about stamping etc -- we seems to be redoing
        # what git-annex does for us already... not really
        r = retry_urlopen(request)
        try:
            r_info = r.info()
            r_stamp = get_url_response_stamp(url, r_info)

            return dict(mtime=r_stamp['mtime'], size=r_stamp['size'], url=url)
        finally:
            r.close()


# TODO:  make it consistent/clear at what stage % encoding/decoding happens!
# now it is a mix!

#
# Useful functionality in requests.models
#  utils.requote_uri -- quote/unquote cycle to guarantee consistent appearance
#  RequestEncodingMixin._encode_params -- Will successfully encode parameters when passed as a dict or a list of ...
#  PreparedRequest().prepare_url(url, params) -- nicely cares about url encodings etc
#

#@auto_repr
# TODO: Well -- it is more of a URI than URL I guess
class URL(object):
    """A helper class to deal with URLs with some "magical" treats to facilitate use of "ssh" urls

    Intended to be a R/O object (i.e. no fields should be changed in-place)

    Although largely decorating urlparse.ParseResult, it
    - doesn't mandate providing all parts of the URL
    - doesn't require netloc but rather asks for separate username, password, and hostname
    - TODO fragment and query are stored as (ordered) dictionaries of single entry elements
      (or otherwise encoded), with empty value being equivalent with no value.  If conversion
      to value=pair fails, string would be stored (so it it was indeed a url pointing to an id
      within a page)

    The idea is that this class should help to break apart a URL, while being
    able to rebuild itself into a string representation for reuse

    If scheme was implicit (e.g. "host:path" for ssh, or later may be "host::path" for
    rsync) ":implicit" suffix added to the scheme (so it is included in the repr of the
    instance "for free")

    Additional semantics we might want to somehow understand/map???
    - strings starting with // (no scheme) assumed to map to local central DataLad dataset,
      with `location` (if not empty) identifying the remote (super)dataset

    Implementation:
    - currently might be overengineered and could be simplified.  Internally keeps
      all values as attributes, but for manipulation in dict (called fields) and converts
      back and forth to ParseResults
    So after we agree on viability of the approach  and collect enough tests we can simplify
    """

    # fields with their defaults
    _FIELDS = (
        'scheme',
        'username',
        'password',
        'hostname', 'port',
        'path',  # 'params',
        'query',
        'fragment',
    )

    __slots__ = _FIELDS + ('_fields', '_str')

    def __init__(self, url=None, **kwargs):
        if url and (bool(url) == bool(kwargs)):
            raise ValueError(
                "Specify either url or breakdown from the fields, not both. "
                "Got url=%r, fields=%r" % (url, kwargs))

        # ok -- let's default to all of them being an empty string
        # Originally used None to signal no value but it complicated
        # operations since often needed first a check before e.g. doing
        # .startswith.  So more harmonic is just to store strings.
        # Not provided seems to be exactly as empty string, so Ok (stdlib
        # guys knew what they were doing ;)
        self._fields = {f: '' for f in self._FIELDS}
        if url:
            self._set_from_str(url)
        else:
            self._set_from_fields(**kwargs)

    @property
    def is_implicit(self):
        return self._fields['scheme'].endswith(':implicit')

    @property
    def fields(self):
        """Returns shallow copy of fields to ease manipulations"""
        return self._fields.copy()

    def __repr__(self):
        # since auto_repr doesn't support "non-0" values atm
        return "%s(%s)" % (
            self.__class__.__name__,
            ", ".join(["%s=%r" % (k, v)
                       for k, v in sorted(self._fields.items())
                       if v]))

    # Some custom __str__s for :implicit URLs
    def __str_ssh__(self):
        """Custom str for ssh:implicit"""
        url = urlunparse(self.to_pr())
        pref = 'ssh:implicit://'
        assert(url.startswith(pref))
        url = url[len(pref):]
        # and we should replace leading /
        url = url.replace('/',
                          ':/' if self.path.startswith('/') else ':',
                          1)
        return url

    def __str_datalad__(self):
        """Custom str for datalad:implicit"""
        fields = self._fields.copy()
        fields['scheme'] = ''
        url = urlunparse(self._fields_to_pr(fields))
        if not fields['hostname']:
            # was of /// type
            url = '//' + url
        return url

    def __str_file__(self):
        """Custom str for datalad:implicit"""
        fields = self._fields.copy()
        fields['scheme'] = ''
        url = urlunparse(self._fields_to_pr(fields))
        return url

    # Lazily evaluated if _str was not set
    def __str__(self):
        if self._str is None:
            self._str = self._as_str()
        return self._str

    def _as_str(self):
        """Re-evaluate string repsentation"""

        if not self.is_implicit:
            return urlunparse(self.to_pr())
        else:
            base_scheme = self.scheme.split(':', 1)[0]
            try:
                __str__ = getattr(self, '__str_%s__' % base_scheme)
            except AttributeError:
                raise ValueError("Don't know how to convert %s:implicit into str"
                                 % base_scheme)
            return __str__()

    #
    # If any field is specified, URL is not considered 'False', i.e.
    # non-existing, although may be we could/shout omit having only
    # scheme or port specified since it doesn't point to any useful
    # location
    #

    def __nonzero__(self):
        fields = self._fields
        return any(fields.values())

    # for PY3
    __bool__ = __nonzero__

    #
    # Helpers to deal with internal structures and conversions
    #

    def _set_from_fields(self, **fields):
        unknown_fields = set(fields).difference(self._FIELDS)
        if unknown_fields:
            raise ValueError("Do not know about %s. Known fields are: %s"
                             % (unknown_fields, self._FIELDS))

        # encode dicts for query or fragment into
        for f in {'query', 'fragment'}:
            v = fields.get(f)
            if isinstance(v, dict):

                ev = urlencode(v)
                # / is reserved char within query
                if f == 'fragment' and '%2F' not in str(v):
                    # but seems to be ok'ish within the fragment which is
                    # the last element of URI and anyways used only by the
                    # client (i.e. by us here if used to compose the URL)
                    # so let's return / back for clarity if there were no
                    # awkward %2F to startwith
                    ev = ev.replace('%2F', '/')
                fields[f] = ev

        self._fields.update(fields)
        self._str = None

    def to_pr(self):
        return self._fields_to_pr(self._fields)

    @classmethod
    def _fields_to_pr(cls, fields):
        """Recompose back fields dict to ParseResult"""
        netloc = fields['username'] or ''
        if fields['password']:
            netloc += ':' + fields['password']
        if netloc:
            netloc += '@'
        netloc += fields['hostname']
        if fields['port']:
            netloc += ':%s' % fields['port']

        pr_fields = {
            f: fields[f]
            for f in cls._FIELDS
            if f not in ('hostname', 'password', 'username', 'port')
        }
        pr_fields['netloc'] = netloc
        pr_fields['params'] = ''

        return ParseResult(**pr_fields)

    def _pr_to_fields(self, pr):
        """ParseResult is a tuple so immutable, which complicates adjusting it

        This function converts ParseResult into dict"""

        if pr.params:
            lgr.warning("ParseResults contains params %r, which will be ignored"
                        % (pr.params,))

        # can't use just pr._asdict since we care to ask those properties
        # such as .port , .hostname etc
        # Forcing '' instead of None since those properties (.hostname), .password,
        # .username return None if not available and we decided to uniformize
        return {f: (getattr(pr, f) or '') for f in self._FIELDS}

    def _set_from_str(self, url):
        fields = self._pr_to_fields(urlparse(url))
        lgr.log(5, "Parsed url %s into fields %s" % (url, fields))

        # Special treatments
        # file:///path should stay file:
        if fields['scheme'] and fields['scheme'] not in {'file'} \
                and not fields['hostname']:
            # dl+archive:... or just for ssh   hostname:path/p1
            if '+' not in fields['scheme']:
                fields['hostname'] = fields['scheme']
                fields['scheme'] = 'ssh:implicit'
                lgr.log(5, "Assuming ssh style url, adjusted: %s" % (fields,))

        if not fields['scheme'] and not fields['hostname']:
            if fields['path'] and '@' in fields['path']:
                # user@host:path/sp1
                fields['scheme'] = 'ssh:implicit'
            elif url.startswith('//'):
                # e.g. // or ///path
                fields['scheme'] = 'datalad:implicit'
            else:
                fields['scheme'] = 'file:implicit'

        if not fields['scheme'] and fields['hostname']:
            # e.g. //a/path
            fields['scheme'] = 'datalad:implicit'

        if fields['scheme'] in ('ssh:implicit',) and not fields['hostname']:
            if fields['query'] or fields['fragment']:
                # actually might be legit with some obscure filenames,
                # so TODO for correct -- probably just re.match the entire url
                # into fields
                raise ValueError("Thought that this url would point to host:path but got query and fragments: %s"
                                 % str(fields))
            #  user@hostname[:path] --> ssh://user@hostname[/path]
            parts = _split_colon(url)
            fields['scheme'] = 'ssh:implicit'
            if len(parts) == 2:
                fields['hostname'] = parts[0]
                fields['path'] = parts[1]
            # hostname might still contain the password
            if fields['hostname'] and '@' in fields['hostname']:
                fields['username'], fields['hostname'] = fields['hostname'].split('@', 1)
            lgr.log(5, "Detected ssh style url, adjusted: %s" % (fields,))

        self._set_from_fields(**fields)
        self._str = url

        # well -- some urls might not unparse identically back
        # strictly speaking, but let's assume they do
        url_ = self._as_str()
        if url != url_:
            lgr.warning("Parsed version of url %r differs from original %r",
                        url_, url)

    #
    # Quick comparators
    #

    def __eq__(self, other):
        if not isinstance(other, URL):
            other = URL(other)
        return other._fields == self._fields

    def __ne__(self, other):
        return not (self == other)

    #
    # Access helpers
    #

    def _parse_qs(self, s, auto_delist=True):
        """Helper around parse_qs to strip unneeded 'list'ing etc and return a dict of key=values"""
        if not s:
            return {}
        out = OrderedDict(parse_qsl(s, 1))
        if not auto_delist:
            return out
        for k in out:
            v = out[k]
            if isinstance(v, list) and len(v) == 1:
                v = v[0]
                out[k] = None if v == '' else v
        return out

    @property
    def query_dict(self):
        return self._parse_qs(self.query)

    @property
    def fragment_dict(self):
        return self._parse_qs(self.fragment)

    # def __getattribute__(self, item):
    #     if item.startswith('_') or item not in URL._FIELDS:
    #         return super(URL, self).__getattribute__(item)
    #     else:
    #         return self._fields[item]

# Bind properties to access fields (without overriding __getattr*)
# This one doesn't work -- I guess due to absent binding of f value
# to the context
#for f in URL._FIELDS:
#   setattr(URL, f, property(lambda self: self._fields[f]))
# These work but ugly duplication
# URL.hostname = property(lambda self: self._fields['hostname'])
# URL.path = property(lambda self: self._fields['path'])
# URL.query = property(lambda self: self._fields['query'])
# URL.scheme = property(lambda self: self._fields['scheme'])
# URL.fragment = property(lambda self: self._fields['fragment'])
# URL.username = property(lambda self: self._fields['username'])
# URL.password = property(lambda self: self._fields['password'])
# URL.port = property(lambda self: self._fields['port'])
for f in URL._FIELDS:
    exec("URL.%s = property(lambda self: self._fields['%s'])" % (f, f))


def _split_colon(s, maxsplit=1):
    """Split on unescaped colon"""
    return re.compile(r'(?<!\\):').split(s, maxsplit=maxsplit)


def parse_url_opts(url):
    """Given a string with url-style query, split into content before # and options as dict"""
    url = URL(url)
    # we need to filter out query and fragment to get the base url
    fields = url.fields
    fields.pop('query')
    fields.pop('fragment')
    opts = url.query_dict
    return str(URL(**fields)), opts


# TODO: should we just define URL.good_for_git or smth like that? ;)
# although git also understands regular paths
def is_url(s):
    """Returns whether a string looks like something what datalad should treat as a URL

    This includes ssh "urls" which git understands.
    """
    try:
        url = URL(s)
    except:
        return False
    implicit = url.is_implicit
    scheme = url.scheme
    return scheme in {'ssh:implicit'} or (not implicit and bool(url))


#### windows workaround ###
# TODO: There should be a better way
def get_local_file_url(fname):
    """Return OS specific URL pointing to a local file

    Parameters
    ----------
    fname : string
        Filename.  If not absolute, abspath is used
    """
    fname = fname if isabs(fname) else abspath(fname)
    if on_windows:
        fname_rep = fname.replace('\\', '/')
        furl = "file:///%s" % urlquote(fname_rep)
        lgr.debug("Replaced '\\' in file\'s url: %s" % furl)
    else:
        # TODO:  need to fix for all the encoding etc
        furl = str(URL(scheme='file', path=urlquote(fname)))
    return furl


def get_local_path_from_url(url):
    """If given a file:// URL, returns a local path, if possible.

    Raises `ValueError` if not possible, for example, if the URL
    scheme is different, or if the `host` isn't empty or 'localhost'

    The returned path is always absolute.
    """
    urlparts = urlsplit(url)
    if not urlparts.scheme == 'file':
        raise ValueError(
            "Non 'file://' URL cannot be resolved to a local path")
    if not (urlparts.netloc in ('', 'localhost', '::1') \
            or urlparts.netloc.startswith('127.')):
        raise ValueError("file:// URL does not point to 'localhost'")
    return urlunquote(urlparts.path)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user