{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Provide access to stuff (html, data files) via HTTP and HTTPS\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport msgpack\nimport os\nimport sys\nimport time\n\nfrom abc import ABCMeta, abstractmethod\nfrom os.path import exists, join as opj, isdir\nfrom six import PY2\nfrom six import binary_type, PY3\nfrom six import add_metaclass\nfrom six import reraise\n\n\nfrom .. import cfg\nfrom ..ui import ui\nfrom ..utils import (\n    auto_repr,\n    unlink,\n)\nfrom ..dochelpers import exc_str\nfrom .credentials import CREDENTIAL_TYPES\nfrom ..support.exceptions import *\nfrom ..support.network import RI\n\nfrom logging import getLogger\nlgr = getLogger('datalad.downloaders')\n\n\n# TODO: remove headers, HTTP specific\n@auto_repr\nclass DownloaderSession(object):\n    \"\"\"Base class to encapsulate information and possibly a session to download the content\n\n    The idea is that corresponding downloader provides all necessary\n    information and if necessary some kind of session to facilitate\n    .download method\n    \"\"\"\n\n    def __init__(self, size=None, filename=None, url=None, headers=None):\n        self.size = size\n        self.filename = filename\n        self.headers = headers\n        self.url = url\n\n    def download(self, f=None, pbar=None, size=None):\n        raise NotImplementedError(\"must be implemented in subclases\")\n\n        # TODO: get_status ?\n\n\n@auto_repr\n@add_metaclass(ABCMeta)\nclass BaseDownloader(object):\n    \"\"\"Base class for the downloaders\"\"\"\n\n    _DEFAULT_AUTHENTICATOR = None\n    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000\n\n    def __init__(self, credential=None, authenticator=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        credential: Credential, optional\n          Provides necessary credential fields to be used by authenticator\n        authenticator: Authenticator, optional\n          Authenticator to use for authentication.\n        \"\"\"\n        if not authenticator and self._DEFAULT_AUTHENTICATOR:\n            authenticator = self._DEFAULT_AUTHENTICATOR()\n\n        if authenticator and authenticator.requires_authentication:\n            if not credential and not authenticator.allows_anonymous:\n                msg = \"Both authenticator and credentials must be provided.\" \\\n                      \" Got only authenticator %s\" % repr(authenticator)\n                if ui.yesno(\n                    title=msg,\n                    text=\"Do you want to enter %s credentials to be used?\" % authenticator.DEFAULT_CREDENTIAL_TYPE\n                ):\n                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](\n                        \"session-only-for-%s\" % id(authenticator))\n                    credential.enter_new()\n                    # TODO: give an option to store those credentials, and generate a basic provider\n                    # record?\n                else:\n                    raise ValueError(msg)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._cache = None  # for fetches, not downloads\n\n    def access(self, method, url, allow_old_session=True, **kwargs):\n        \"\"\"Generic decorator to manage access to the URL via some method\n\n        Parameters\n        ----------\n        method : callable\n          A callable, usually a method of the same class, which we decorate\n          with access handling, and pass url as the first argument\n        url : string\n          URL to access\n        *args, **kwargs\n          Passed into the method call\n\n        Returns\n        -------\n        None or bytes\n        \"\"\"\n        # TODO: possibly wrap this logic outside within a decorator, which\n        # would just call the corresponding method\n        authenticator = self.authenticator\n        if authenticator:\n            needs_authentication = authenticator.requires_authentication\n        else:\n            needs_authentication = self.credential\n\n        attempt, incomplete_attempt = 0, 0\n        while True:\n            attempt += 1\n            if attempt > 20:\n                # are we stuck in a loop somehow? I think logic doesn't allow this atm\n                raise RuntimeError(\"Got to the %d'th iteration while trying to download %s\" % (attempt, url))\n            exc_info = None\n            supported_auth_types = []\n            try:\n                used_old_session = False\n                access_denied = False\n                used_old_session = self._establish_session(url, allow_old=allow_old_session)\n                if not allow_old_session:\n                    assert(not used_old_session)\n                lgr.log(5, \"Calling out into %s for %s\" % (method, url))\n                result = method(url, **kwargs)\n                # assume success if no puke etc\n                break\n            except AccessDeniedError as e:\n                if isinstance(e, AnonymousAccessDeniedError):\n                    access_denied = \"Anonymous\"\n                else:\n                    access_denied = \"Authenticated\"\n                lgr.debug(\"%s access was denied: %s\", access_denied, exc_str(e))\n                supported_auth_types = e.supported_types\n                exc_info = sys.exc_info()\n\n            except IncompleteDownloadError as e:\n                exc_info = sys.exc_info()\n                incomplete_attempt += 1\n                if incomplete_attempt > 5:\n                    # give up\n                    raise\n                lgr.debug(\"Failed to download fully, will try again: %s\", exc_str(e))\n                # TODO: may be fail ealier than after 20 attempts in such a case?\n            except DownloadError:\n                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!\n                raise\n\n            msg_types = ''\n            if supported_auth_types:\n                msg_types = \" The failure response indicated that following \" \\\n                            \"authentication types should be used: %s\" % (', '.join(supported_auth_types))\n            if access_denied:  # moved logic outside of except for clarity\n                # TODO: what if it was anonimous attempt without authentication,\n                #     so it is not \"requires_authentication\" but rather\n                #     \"supports_authentication\"?  We should not report below in\n                # _get_new_credential that authentication has failed then since there\n                # were no authentication.  We might need a custom exception to\n                # be caught above about that\n                if needs_authentication:\n                    # so we knew it needs authentication\n                    if used_old_session:\n                        # Let's try with fresh ones\n                        allow_old_session = False\n                        continue\n                    else:\n                        # we did use new cookies, we knew that authentication is needed\n                        # but still failed. So possible cases:\n                        #  1. authentication credentials changed/were revoked\n                        #     - allow user to re-enter credentials\n                        #  2. authentication mechanisms changed\n                        #     - we can't do anything here about that\n                        #  3. bug in out code which would render authentication/cookie handling\n                        #     ineffective\n                        #     - not sure what to do about it\n                        if not ui.is_interactive:\n                            lgr.error(\n                                \"Interface is non interactive, so we are \"\n                                \"reraising: %s\" % exc_str(e))\n                            reraise(*exc_info)\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=False)\n                        allow_old_session = False\n                        continue\n                else:  # None or False\n                    if needs_authentication is False:\n                        # those urls must or should NOT require authentication\n                        # but we got denied\n                        raise DownloadError(\n                            \"Failed to download from %s, which must be available\"\n                            \"without authentication but access was denied. \"\n                            \"Adjust your configuration for the provider.%s\"\n                            % (url, msg_types))\n                    else:\n                        # how could be None or any other non-False bool(False)\n                        assert(needs_authentication is None)\n                        # So we didn't know if authentication necessary, and it\n                        # seems to be necessary, so Let's ask the user to setup\n                        # authentication mechanism for this website\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=True)\n                        allow_old_session = False\n                        continue\n\n        return result\n\n    def _setup_new_provider(self, title, url, auth_types=None):\n        # Full new provider (TODO move into Providers?)\n        from .providers import Providers\n        providers = Providers.from_config_files()\n        while True:\n            provider = providers.enter_new(url, auth_types=auth_types)\n            if not provider:\n                if ui.yesno(\n                    title=\"Re-enter provider?\",\n                    text=\"You haven't entered or saved provider, would you like to retry?\",\n                    default=True\n                ):\n                    continue\n            break\n        return provider\n\n\n    def _enter_credentials(\n            self, url, denied_msg,\n            auth_types=[], new_provider=True):\n        \"\"\"Use when authentication fails to set new credentials for url\n\n        Raises\n        ------\n        DownloadError\n          If no known credentials type or user refuses to update\n        \"\"\"\n        title = \"{msg} access to {url} has failed.\".format(\n            msg=denied_msg, url=url)\n\n        if new_provider:\n            # No credential was known, we need to create an\n            # appropriate one\n            if not ui.yesno(\n                    title=title,\n                    text=\"Would you like to setup a new provider configuration\"\n                         \" to access url?\",\n                    default=True\n            ):\n                assert not self.authenticator, \"bug: incorrect assumption\"\n                raise DownloadError(\n                    title +\n                    \" No authenticator is known, cannot set any credential\")\n            else:\n                provider = self._setup_new_provider(\n                    title, url, auth_types=auth_types)\n                self.authenticator = provider.authenticator\n                self.credential = provider.credential\n                if not (self.credential and self.credential.is_known):\n                    # TODO: or should we ask to re-enter?\n                    self.credential.enter_new()\n        else:\n            action_msg = \"enter other credentials in case they were updated?\"\n\n            if ui.yesno(\n                    title=title,\n                    text=\"Do you want to %s\" % action_msg):\n                self.credential.enter_new()\n            else:\n                raise DownloadError(\n                    \"Failed to download from %s given available credentials\"\n                    % url)\n\n    @staticmethod\n    def _get_temp_download_filename(filepath):\n        \"\"\"Given a filepath, return the one to use as temp file during download\n        \"\"\"\n        # TODO: might better reside somewhere under .datalad/tmp or\n        # .git/datalad/tmp\n        return filepath + \".datalad-download-temp\"\n\n    @abstractmethod\n    def get_downloader_session(self, url):\n        \"\"\"\n\n        Parameters\n        ----------\n        url : str\n\n        Returns\n        -------\n        downloader_into_fp: callable\n           Which takes two parameters: file, pbar\n        target_size: int or None (if unknown)\n        url: str\n           Possibly redirected url\n        url_filename: str or None\n           Filename as decided from the (redirected) url\n        headers : dict or None\n        \"\"\"\n        raise NotImplementedError(\"Must be implemented in the subclass\")\n\n    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):\n        \"\"\"Verify that download finished correctly\"\"\"\n\n        if (self.authenticator\n                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \\\n                and hasattr(self.authenticator, 'failure_re') \\\n                and self.authenticator.failure_re:\n            assert hasattr(self.authenticator, 'check_for_auth_failure'), \\\n                \"%s has failure_re defined but no check_for_auth_failure\" \\\n                % self.authenticator\n\n            if file_:\n                with open(file_) as fp:\n                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)\n            else:\n                assert(content is not None)\n\n            self.authenticator.check_for_auth_failure(\n                content, \"Download of the url %s has failed: \" % url)\n\n        if target_size and target_size != downloaded_size:\n            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(\n                \"Downloaded size %d differs from originally announced %d\" % (downloaded_size, target_size))\n\n    def _download(self, url, path=None, overwrite=False, size=None, stats=None):\n        \"\"\"Download content into a file\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        path: str, optional\n          Path to file where to store the downloaded content.  If None,\n          filename deduced from the url and saved in curdir\n        size: int, optional\n          Limit in size to be downloaded\n\n        Returns\n        -------\n        None or string\n          Returns downloaded filename\n\n        \"\"\"\n\n        downloader_session = self.get_downloader_session(url)\n        status = self.get_status_from_headers(downloader_session.headers)\n\n        target_size = downloader_session.size\n        if size is not None:\n            target_size = min(target_size, size)\n\n        #### Specific to download\n        if path:\n            if isdir(path):\n                # provided path is a directory under which to save\n                filename = downloader_session.filename\n                if not filename:\n                    raise DownloadError(\n                        \"File name could not be determined from {}\".format(url))\n                filepath = opj(path, filename)\n            else:\n                filepath = path\n        else:\n            filepath = downloader_session.filename\n\n        existed = exists(filepath)\n        if existed and not overwrite:\n            raise DownloadError(\"File %s already exists\" % filepath)\n\n        # FETCH CONTENT\n        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])\n        try:\n            temp_filepath = self._get_temp_download_filename(filepath)\n            if exists(temp_filepath):\n                # eventually we might want to continue the download\n                lgr.warning(\n                    \"Temporary file %s from the previous download was found. \"\n                    \"It will be overriden\" % temp_filepath)\n                # TODO.  also logic below would clean it up atm\n\n            with open(temp_filepath, 'wb') as fp:\n                # TODO: url might be a bit too long for the beast.\n                # Consider to improve to make it animated as well, or shorten here\n                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n                t0 = time.time()\n                downloader_session.download(fp, pbar, size=size)\n                downloaded_time = time.time() - t0\n                pbar.finish()\n            downloaded_size = os.stat(temp_filepath).st_size\n\n            # (headers.get('Content-type', \"\") and headers.get('Content-Type')).startswith('text/html')\n            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator\n            self._verify_download(url, downloaded_size, target_size, temp_filepath)\n\n            # adjust atime/mtime according to headers/status\n            if status.mtime:\n                lgr.log(5, \"Setting mtime for %s to be %s\", temp_filepath, status.mtime)\n                os.utime(temp_filepath, (time.time(), status.mtime))\n\n            # place successfully downloaded over the filepath\n            os.rename(temp_filepath, filepath)\n\n            if stats:\n                stats.downloaded += 1\n                stats.overwritten += int(existed)\n                stats.downloaded_size += downloaded_size\n                stats.downloaded_time += downloaded_time\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to download {url} into {filepath}: {e_str}\".format(\n                **locals()\n            ))\n            raise DownloadError(exc_str(e))  # for now\n        finally:\n            if exists(temp_filepath):\n                # clean up\n                lgr.debug(\"Removing a temporary download %s\", temp_filepath)\n                unlink(temp_filepath)\n\n        return filepath\n\n    def download(self, url, path=None, **kwargs):\n        \"\"\"Fetch content as pointed by the URL optionally into a file\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        path : string, optional\n          Filename or existing directory to store downloaded content under.\n          If not provided -- deduced from the url\n\n        Returns\n        -------\n        string\n          file path\n        \"\"\"\n        # TODO: may be move all the path dealing logic here\n        # but then it might require sending request anyways for Content-Disposition\n        # so probably nah\n        lgr.info(\"Downloading %r into %r\", url, path)\n        return self.access(self._download, url, path=path, **kwargs)\n\n    @property\n    def cache(self):\n        if self._cache is None:\n            # TODO: move this all logic outside into a dedicated caching beast\n            lgr.info(\"Initializing cache for fetches\")\n            if PY2:\n                import anydbm as dbm\n            else:\n                import dbm\n            # Initiate cache.\n            # Very rudimentary caching for now, might fail many ways\n            cache_dir = cfg.obtain('datalad.locations.cache')\n            if not exists(cache_dir):\n                os.makedirs(cache_dir)\n            cache_path = opj(cache_dir, 'crawl_cache.dbm')\n            self._cache = dbm.open(cache_path, 'c')\n            import atexit\n            atexit.register(self._cache.close)\n        return self._cache\n\n    def _fetch(self, url, cache=None, size=None, allow_redirects=True):\n        \"\"\"Fetch content from a url into a file.\n\n        Very similar to _download but lacks any \"file\" management and decodes\n        content\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        cache: bool, optional\n          If None, config is consulted to determine whether results should be\n          cached. Cache is operating based on url, so no verification of any\n          kind is carried out\n\n        Returns\n        -------\n        bytes, dict\n          content, headers\n        \"\"\"\n        lgr.log(3, \"_fetch(%r, cache=%r, size=%r, allow_redirects=%r)\",\n                url, cache, size, allow_redirects)\n        if cache is None:\n            cache = cfg.obtain('datalad.crawl.cache', default=False)\n\n        if cache:\n            cache_key = msgpack.dumps(url)\n            lgr.debug(\"Loading content for url %s from cache\", url)\n            res = self.cache.get(cache_key)\n            if res is not None:\n                try:\n                    return msgpack.loads(res, encoding='utf-8')\n                except Exception as exc:\n                    lgr.warning(\"Failed to unpack loaded from cache for %s: %s\",\n                                url, exc_str(exc))\n\n        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)\n\n        target_size = downloader_session.size\n        if size is not None:\n            if size == 0:\n                # no download of the content was requested -- just return headers and be done\n                return None, downloader_session.headers\n            target_size = min(size, target_size)\n\n        # FETCH CONTENT\n        try:\n            # Consider to improve to make it animated as well, or shorten here\n            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n            content = downloader_session.download(size=size)\n            #pbar.finish()\n            downloaded_size = len(content)\n\n            # now that we know size based on encoded content, let's decode into string type\n            if PY3 and isinstance(content, binary_type):\n                content = content.decode()\n            # downloaded_size = os.stat(temp_filepath).st_size\n\n            self._verify_download(url, downloaded_size, target_size, None, content=content)\n\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to fetch {url}: {e_str}\".format(**locals()))\n            raise DownloadError(exc_str(e, limit=8))  # for now\n\n        if cache:\n            # apparently requests' CaseInsensitiveDict is not serialazable\n            # TODO:  may be we should reuse that type everywhere, to avoid\n            # out own handling for case-handling\n            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))\n\n        return content, downloader_session.headers\n\n    def fetch(self, url, **kwargs):\n        \"\"\"Fetch and return content (not decoded) as pointed by the URL\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        bytes\n          content\n        \"\"\"\n        lgr.info(\"Fetching %r\", url)\n        # Do not return headers, just content\n        out = self.access(self._fetch, url, **kwargs)\n        return out[0]\n\n    def get_status(self, url, old_status=None, **kwargs):\n        \"\"\"Return status of the url as a dict, None if N/A\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        old_status : FileStatus, optional\n          Previous status record.  If provided, might serve as a shortcut\n          to assess if status has changed, and if not -- return the same\n          record\n\n        Returns\n        -------\n        dict\n          dict-like beast depicting the status of the URL if accessible.\n          Returned value should be sufficient to tell if the URL content\n          has changed by comparing to previously obtained value.\n          If URL is not reachable, None would be returned\n        \"\"\"\n        return self.access(self._get_status, url, old_status=old_status, **kwargs)\n\n    # TODO: borrow from itself... ?\n    # @borrowkwargs(BaseDownloader, 'get_status')\n    def _get_status(self, url, old_status=None):\n\n        # the tricky part is only to make sure that we are getting the target URL\n        # and not some page saying to login, that is why we need to fetch some content\n        # in those cases, and not just check the headers\n        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \\\n            if self.authenticator \\\n            and hasattr(self.authenticator, 'failure_re') \\\n            and self.authenticator.failure_re \\\n            else 0\n\n        _, headers = self._fetch(url, cache=False, size=download_size)\n\n        # extract from headers information to depict the status of the url\n        status = self.get_status_from_headers(headers)\n\n        if old_status is not None:\n            raise NotImplementedError(\"Do not know yet how to deal with old_status. TODO\")\n\n        return status\n\n    @classmethod\n    @abstractmethod\n    def get_status_from_headers(cls, headers):\n        raise NotImplementedError(\"Implement in the subclass: %s\" % cls)\n\n    def get_target_url(self, url):\n        \"\"\"Return url after possible redirections\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return self.access(self._get_target_url, url)\n\n    def _get_target_url(self, url):\n        return self.get_downloader_session(url).url\n\n\n#\n# Authenticators    XXX might go into authenticators.py\n#\n\nclass Authenticator(object):\n    \"\"\"Abstract common class for different types of authentication\n\n    Derived classes should get parameterized with options from the config files\n    from \"provider:\" sections\n    \"\"\"\n    requires_authentication = True\n    allows_anonymous = False\n    # TODO: figure out interface\n\n    DEFAULT_CREDENTIAL_TYPE = 'user_password'\n\n    def authenticate(self, *args, **kwargs):\n        \"\"\"Derived classes will provide specific implementation\n        \"\"\"\n        if self.requires_authentication:\n            raise NotImplementedError(\"Authentication for %s not yet implemented\" % self.__class__)\n\n\nclass NotImplementedAuthenticator(Authenticator):\n    pass\n\n\nclass NoneAuthenticator(Authenticator):\n    \"\"\"Whenever no authentication is necessary and that is stated explicitly\"\"\"\n    requires_authentication = False\n    pass\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Provide access to stuff (html, data files) via HTTP and HTTPS\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport msgpack\nimport os\nimport sys\nimport time\n\nfrom abc import ABCMeta, abstractmethod\nfrom os.path import exists, join as opj, isdir\nfrom six import PY2\nfrom six import binary_type, PY3\nfrom six import add_metaclass\nfrom six import reraise\n\n\nfrom .. import cfg\nfrom ..ui import ui\nfrom ..utils import (\n    auto_repr,\n    unlink,\n)\nfrom ..dochelpers import exc_str\nfrom .credentials import CREDENTIAL_TYPES\nfrom ..support.exceptions import *\nfrom ..support.network import RI\n\nfrom logging import getLogger\nlgr = getLogger('datalad.downloaders')\n\n\n# TODO: remove headers, HTTP specific\n@auto_repr\nclass DownloaderSession(object):\n    \"\"\"Base class to encapsulate information and possibly a session to download the content\n\n    The idea is that corresponding downloader provides all necessary\n    information and if necessary some kind of session to facilitate\n    .download method\n    \"\"\"\n\n    def __init__(self, size=None, filename=None, url=None, headers=None):\n        self.size = size\n        self.filename = filename\n        self.headers = headers\n        self.url = url\n\n    def download(self, f=None, pbar=None, size=None):\n        raise NotImplementedError(\"must be implemented in subclases\")\n\n        # TODO: get_status ?\n\n\n@auto_repr\n@add_metaclass(ABCMeta)\nclass BaseDownloader(object):\n    \"\"\"Base class for the downloaders\"\"\"\n\n    _DEFAULT_AUTHENTICATOR = None\n    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000\n\n    def __init__(self, credential=None, authenticator=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        credential: Credential, optional\n          Provides necessary credential fields to be used by authenticator\n        authenticator: Authenticator, optional\n          Authenticator to use for authentication.\n        \"\"\"\n        if not authenticator and self._DEFAULT_AUTHENTICATOR:\n            authenticator = self._DEFAULT_AUTHENTICATOR()\n\n        if authenticator and authenticator.requires_authentication:\n            if not credential and not authenticator.allows_anonymous:\n                msg = \"Both authenticator and credentials must be provided.\" \\\n                      \" Got only authenticator %s\" % repr(authenticator)\n                if ui.yesno(\n                    title=msg,\n                    text=\"Do you want to enter %s credentials to be used?\" % authenticator.DEFAULT_CREDENTIAL_TYPE\n                ):\n                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](\n                        \"session-only-for-%s\" % id(authenticator))\n                    credential.enter_new()\n                    # TODO: give an option to store those credentials, and generate a basic provider\n                    # record?\n                else:\n                    raise ValueError(msg)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._cache = None  # for fetches, not downloads\n\n    def access(self, method, url, allow_old_session=True, **kwargs):\n        \"\"\"Generic decorator to manage access to the URL via some method\n\n        Parameters\n        ----------\n        method : callable\n          A callable, usually a method of the same class, which we decorate\n          with access handling, and pass url as the first argument\n        url : string\n          URL to access\n        *args, **kwargs\n          Passed into the method call\n\n        Returns\n        -------\n        None or bytes\n        \"\"\"\n        # TODO: possibly wrap this logic outside within a decorator, which\n        # would just call the corresponding method\n        authenticator = self.authenticator\n        if authenticator:\n            needs_authentication = authenticator.requires_authentication\n        else:\n            needs_authentication = self.credential\n\n        attempt, incomplete_attempt = 0, 0\n        while True:\n            attempt += 1\n            if attempt > 20:\n                # are we stuck in a loop somehow? I think logic doesn't allow this atm\n                raise RuntimeError(\"Got to the %d'th iteration while trying to download %s\" % (attempt, url))\n            exc_info = None\n            supported_auth_types = []\n            try:\n                used_old_session = False\n                access_denied = False\n                used_old_session = self._establish_session(url, allow_old=allow_old_session)\n                if not allow_old_session:\n                    assert(not used_old_session)\n                lgr.log(5, \"Calling out into %s for %s\" % (method, url))\n                result = method(url, **kwargs)\n                # assume success if no puke etc\n                break\n            except AccessDeniedError as e:\n                if isinstance(e, AnonymousAccessDeniedError):\n                    access_denied = \"Anonymous\"\n                else:\n                    access_denied = \"Authenticated\"\n                lgr.debug(\"%s access was denied: %s\", access_denied, exc_str(e))\n                supported_auth_types = e.supported_types\n                exc_info = sys.exc_info()\n\n            except IncompleteDownloadError as e:\n                exc_info = sys.exc_info()\n                incomplete_attempt += 1\n                if incomplete_attempt > 5:\n                    # give up\n                    raise\n                lgr.debug(\"Failed to download fully, will try again: %s\", exc_str(e))\n                # TODO: may be fail ealier than after 20 attempts in such a case?\n            except DownloadError:\n                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!\n                raise\n\n            msg_types = ''\n            if supported_auth_types:\n                msg_types = \" The failure response indicated that following \" \\\n                            \"authentication types should be used: %s\" % (', '.join(supported_auth_types))\n            if access_denied:  # moved logic outside of except for clarity\n                # TODO: what if it was anonimous attempt without authentication,\n                #     so it is not \"requires_authentication\" but rather\n                #     \"supports_authentication\"?  We should not report below in\n                # _get_new_credential that authentication has failed then since there\n                # were no authentication.  We might need a custom exception to\n                # be caught above about that\n                if needs_authentication:\n                    # so we knew it needs authentication\n                    if used_old_session:\n                        # Let's try with fresh ones\n                        allow_old_session = False\n                        continue\n                    else:\n                        # we did use new cookies, we knew that authentication is needed\n                        # but still failed. So possible cases:\n                        #  1. authentication credentials changed/were revoked\n                        #     - allow user to re-enter credentials\n                        #  2. authentication mechanisms changed\n                        #     - we can't do anything here about that\n                        #  3. bug in out code which would render authentication/cookie handling\n                        #     ineffective\n                        #     - not sure what to do about it\n                        if not ui.is_interactive:\n                            lgr.error(\n                                \"Interface is non interactive, so we are \"\n                                \"reraising: %s\" % exc_str(e))\n                            reraise(*exc_info)\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=False)\n                        allow_old_session = False\n                        continue\n                else:  # None or False\n                    if needs_authentication is False:\n                        # those urls must or should NOT require authentication\n                        # but we got denied\n                        raise DownloadError(\n                            \"Failed to download from %s, which must be available\"\n                            \"without authentication but access was denied. \"\n                            \"Adjust your configuration for the provider.%s\"\n                            % (url, msg_types))\n                    else:\n                        # how could be None or any other non-False bool(False)\n                        assert(needs_authentication is None)\n                        # So we didn't know if authentication necessary, and it\n                        # seems to be necessary, so Let's ask the user to setup\n                        # authentication mechanism for this website\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=True)\n                        allow_old_session = False\n                        continue\n\n        return result\n\n    def _setup_new_provider(self, title, url, auth_types=None):\n        # Full new provider (TODO move into Providers?)\n        from .providers import Providers\n        providers = Providers.from_config_files()\n        while True:\n            provider = providers.enter_new(url, auth_types=auth_types)\n            if not provider:\n                if ui.yesno(\n                    title=\"Re-enter provider?\",\n                    text=\"You haven't entered or saved provider, would you like to retry?\",\n                    default=True\n                ):\n                    continue\n            break\n        return provider\n\n\n    def _enter_credentials(\n            self, url, denied_msg,\n            auth_types=[], new_provider=True):\n        \"\"\"Use when authentication fails to set new credentials for url\n\n        Raises\n        ------\n        DownloadError\n          If no known credentials type or user refuses to update\n        \"\"\"\n        title = \"{msg} access to {url} has failed.\".format(\n            msg=denied_msg, url=url)\n\n        if new_provider:\n            # No credential was known, we need to create an\n            # appropriate one\n            if not ui.yesno(\n                    title=title,\n                    text=\"Would you like to setup a new provider configuration\"\n                         \" to access url?\",\n                    default=True\n            ):\n                assert not self.authenticator, \"bug: incorrect assumption\"\n                raise DownloadError(\n                    title +\n                    \" No authenticator is known, cannot set any credential\")\n            else:\n                provider = self._setup_new_provider(\n                    title, url, auth_types=auth_types)\n                self.authenticator = provider.authenticator\n                self.credential = provider.credential\n                if not (self.credential and self.credential.is_known):\n                    # TODO: or should we ask to re-enter?\n                    self.credential.enter_new()\n        else:\n            action_msg = \"enter other credentials in case they were updated?\"\n\n            if self.credential and ui.yesno(\n                    title=title,\n                    text=\"Do you want to %s\" % action_msg):\n                self.credential.enter_new()\n            else:\n                raise DownloadError(\n                    \"Failed to download from %s given available credentials\"\n                    % url)\n\n    @staticmethod\n    def _get_temp_download_filename(filepath):\n        \"\"\"Given a filepath, return the one to use as temp file during download\n        \"\"\"\n        # TODO: might better reside somewhere under .datalad/tmp or\n        # .git/datalad/tmp\n        return filepath + \".datalad-download-temp\"\n\n    @abstractmethod\n    def get_downloader_session(self, url):\n        \"\"\"\n\n        Parameters\n        ----------\n        url : str\n\n        Returns\n        -------\n        downloader_into_fp: callable\n           Which takes two parameters: file, pbar\n        target_size: int or None (if unknown)\n        url: str\n           Possibly redirected url\n        url_filename: str or None\n           Filename as decided from the (redirected) url\n        headers : dict or None\n        \"\"\"\n        raise NotImplementedError(\"Must be implemented in the subclass\")\n\n    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):\n        \"\"\"Verify that download finished correctly\"\"\"\n\n        if (self.authenticator\n                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \\\n                and hasattr(self.authenticator, 'failure_re') \\\n                and self.authenticator.failure_re:\n            assert hasattr(self.authenticator, 'check_for_auth_failure'), \\\n                \"%s has failure_re defined but no check_for_auth_failure\" \\\n                % self.authenticator\n\n            if file_:\n                with open(file_) as fp:\n                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)\n            else:\n                assert(content is not None)\n\n            self.authenticator.check_for_auth_failure(\n                content, \"Download of the url %s has failed: \" % url)\n\n        if target_size and target_size != downloaded_size:\n            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(\n                \"Downloaded size %d differs from originally announced %d\" % (downloaded_size, target_size))\n\n    def _download(self, url, path=None, overwrite=False, size=None, stats=None):\n        \"\"\"Download content into a file\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        path: str, optional\n          Path to file where to store the downloaded content.  If None,\n          filename deduced from the url and saved in curdir\n        size: int, optional\n          Limit in size to be downloaded\n\n        Returns\n        -------\n        None or string\n          Returns downloaded filename\n\n        \"\"\"\n\n        downloader_session = self.get_downloader_session(url)\n        status = self.get_status_from_headers(downloader_session.headers)\n\n        target_size = downloader_session.size\n        if size is not None:\n            target_size = min(target_size, size)\n\n        #### Specific to download\n        if path:\n            if isdir(path):\n                # provided path is a directory under which to save\n                filename = downloader_session.filename\n                if not filename:\n                    raise DownloadError(\n                        \"File name could not be determined from {}\".format(url))\n                filepath = opj(path, filename)\n            else:\n                filepath = path\n        else:\n            filepath = downloader_session.filename\n\n        existed = exists(filepath)\n        if existed and not overwrite:\n            raise DownloadError(\"File %s already exists\" % filepath)\n\n        # FETCH CONTENT\n        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])\n        try:\n            temp_filepath = self._get_temp_download_filename(filepath)\n            if exists(temp_filepath):\n                # eventually we might want to continue the download\n                lgr.warning(\n                    \"Temporary file %s from the previous download was found. \"\n                    \"It will be overriden\" % temp_filepath)\n                # TODO.  also logic below would clean it up atm\n\n            with open(temp_filepath, 'wb') as fp:\n                # TODO: url might be a bit too long for the beast.\n                # Consider to improve to make it animated as well, or shorten here\n                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n                t0 = time.time()\n                downloader_session.download(fp, pbar, size=size)\n                downloaded_time = time.time() - t0\n                pbar.finish()\n            downloaded_size = os.stat(temp_filepath).st_size\n\n            # (headers.get('Content-type', \"\") and headers.get('Content-Type')).startswith('text/html')\n            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator\n            self._verify_download(url, downloaded_size, target_size, temp_filepath)\n\n            # adjust atime/mtime according to headers/status\n            if status.mtime:\n                lgr.log(5, \"Setting mtime for %s to be %s\", temp_filepath, status.mtime)\n                os.utime(temp_filepath, (time.time(), status.mtime))\n\n            # place successfully downloaded over the filepath\n            os.rename(temp_filepath, filepath)\n\n            if stats:\n                stats.downloaded += 1\n                stats.overwritten += int(existed)\n                stats.downloaded_size += downloaded_size\n                stats.downloaded_time += downloaded_time\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to download {url} into {filepath}: {e_str}\".format(\n                **locals()\n            ))\n            raise DownloadError(exc_str(e))  # for now\n        finally:\n            if exists(temp_filepath):\n                # clean up\n                lgr.debug(\"Removing a temporary download %s\", temp_filepath)\n                unlink(temp_filepath)\n\n        return filepath\n\n    def download(self, url, path=None, **kwargs):\n        \"\"\"Fetch content as pointed by the URL optionally into a file\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        path : string, optional\n          Filename or existing directory to store downloaded content under.\n          If not provided -- deduced from the url\n\n        Returns\n        -------\n        string\n          file path\n        \"\"\"\n        # TODO: may be move all the path dealing logic here\n        # but then it might require sending request anyways for Content-Disposition\n        # so probably nah\n        lgr.info(\"Downloading %r into %r\", url, path)\n        return self.access(self._download, url, path=path, **kwargs)\n\n    @property\n    def cache(self):\n        if self._cache is None:\n            # TODO: move this all logic outside into a dedicated caching beast\n            lgr.info(\"Initializing cache for fetches\")\n            if PY2:\n                import anydbm as dbm\n            else:\n                import dbm\n            # Initiate cache.\n            # Very rudimentary caching for now, might fail many ways\n            cache_dir = cfg.obtain('datalad.locations.cache')\n            if not exists(cache_dir):\n                os.makedirs(cache_dir)\n            cache_path = opj(cache_dir, 'crawl_cache.dbm')\n            self._cache = dbm.open(cache_path, 'c')\n            import atexit\n            atexit.register(self._cache.close)\n        return self._cache\n\n    def _fetch(self, url, cache=None, size=None, allow_redirects=True):\n        \"\"\"Fetch content from a url into a file.\n\n        Very similar to _download but lacks any \"file\" management and decodes\n        content\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        cache: bool, optional\n          If None, config is consulted to determine whether results should be\n          cached. Cache is operating based on url, so no verification of any\n          kind is carried out\n\n        Returns\n        -------\n        bytes, dict\n          content, headers\n        \"\"\"\n        lgr.log(3, \"_fetch(%r, cache=%r, size=%r, allow_redirects=%r)\",\n                url, cache, size, allow_redirects)\n        if cache is None:\n            cache = cfg.obtain('datalad.crawl.cache', default=False)\n\n        if cache:\n            cache_key = msgpack.dumps(url)\n            lgr.debug(\"Loading content for url %s from cache\", url)\n            res = self.cache.get(cache_key)\n            if res is not None:\n                try:\n                    return msgpack.loads(res, encoding='utf-8')\n                except Exception as exc:\n                    lgr.warning(\"Failed to unpack loaded from cache for %s: %s\",\n                                url, exc_str(exc))\n\n        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)\n\n        target_size = downloader_session.size\n        if size is not None:\n            if size == 0:\n                # no download of the content was requested -- just return headers and be done\n                return None, downloader_session.headers\n            target_size = min(size, target_size)\n\n        # FETCH CONTENT\n        try:\n            # Consider to improve to make it animated as well, or shorten here\n            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n            content = downloader_session.download(size=size)\n            #pbar.finish()\n            downloaded_size = len(content)\n\n            # now that we know size based on encoded content, let's decode into string type\n            if PY3 and isinstance(content, binary_type):\n                content = content.decode()\n            # downloaded_size = os.stat(temp_filepath).st_size\n\n            self._verify_download(url, downloaded_size, target_size, None, content=content)\n\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to fetch {url}: {e_str}\".format(**locals()))\n            raise DownloadError(exc_str(e, limit=8))  # for now\n\n        if cache:\n            # apparently requests' CaseInsensitiveDict is not serialazable\n            # TODO:  may be we should reuse that type everywhere, to avoid\n            # out own handling for case-handling\n            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))\n\n        return content, downloader_session.headers\n\n    def fetch(self, url, **kwargs):\n        \"\"\"Fetch and return content (not decoded) as pointed by the URL\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        bytes\n          content\n        \"\"\"\n        lgr.info(\"Fetching %r\", url)\n        # Do not return headers, just content\n        out = self.access(self._fetch, url, **kwargs)\n        return out[0]\n\n    def get_status(self, url, old_status=None, **kwargs):\n        \"\"\"Return status of the url as a dict, None if N/A\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        old_status : FileStatus, optional\n          Previous status record.  If provided, might serve as a shortcut\n          to assess if status has changed, and if not -- return the same\n          record\n\n        Returns\n        -------\n        dict\n          dict-like beast depicting the status of the URL if accessible.\n          Returned value should be sufficient to tell if the URL content\n          has changed by comparing to previously obtained value.\n          If URL is not reachable, None would be returned\n        \"\"\"\n        return self.access(self._get_status, url, old_status=old_status, **kwargs)\n\n    # TODO: borrow from itself... ?\n    # @borrowkwargs(BaseDownloader, 'get_status')\n    def _get_status(self, url, old_status=None):\n\n        # the tricky part is only to make sure that we are getting the target URL\n        # and not some page saying to login, that is why we need to fetch some content\n        # in those cases, and not just check the headers\n        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \\\n            if self.authenticator \\\n            and hasattr(self.authenticator, 'failure_re') \\\n            and self.authenticator.failure_re \\\n            else 0\n\n        _, headers = self._fetch(url, cache=False, size=download_size)\n\n        # extract from headers information to depict the status of the url\n        status = self.get_status_from_headers(headers)\n\n        if old_status is not None:\n            raise NotImplementedError(\"Do not know yet how to deal with old_status. TODO\")\n\n        return status\n\n    @classmethod\n    @abstractmethod\n    def get_status_from_headers(cls, headers):\n        raise NotImplementedError(\"Implement in the subclass: %s\" % cls)\n\n    def get_target_url(self, url):\n        \"\"\"Return url after possible redirections\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return self.access(self._get_target_url, url)\n\n    def _get_target_url(self, url):\n        return self.get_downloader_session(url).url\n\n\n#\n# Authenticators    XXX might go into authenticators.py\n#\n\nclass Authenticator(object):\n    \"\"\"Abstract common class for different types of authentication\n\n    Derived classes should get parameterized with options from the config files\n    from \"provider:\" sections\n    \"\"\"\n    requires_authentication = True\n    allows_anonymous = False\n    # TODO: figure out interface\n\n    DEFAULT_CREDENTIAL_TYPE = 'user_password'\n\n    def authenticate(self, *args, **kwargs):\n        \"\"\"Derived classes will provide specific implementation\n        \"\"\"\n        if self.requires_authentication:\n            raise NotImplementedError(\"Authentication for %s not yet implemented\" % self.__class__)\n\n\nclass NotImplementedAuthenticator(Authenticator):\n    pass\n\n\nclass NoneAuthenticator(Authenticator):\n    \"\"\"Whenever no authentication is necessary and that is stated explicitly\"\"\"\n    requires_authentication = False\n    pass\n",
            "header": {
                "file": {
                    "path": "datalad/downloaders/base.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "37e09fa25",
                "new_commit_hash": "eafe16a70",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/downloaders/base.py"
            },
            "plus_file": {
                "path": "b/datalad/downloaders/base.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 284,
                            "length": 7
                        },
                        "new_range": {
                            "start": 284,
                            "length": 7
                        },
                        "section": "class BaseDownloader(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        else:",
                                "type": "context"
                            },
                            {
                                "content": "            action_msg = \"enter other credentials in case they were updated?\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "            if ui.yesno(",
                                "type": "deleted"
                            },
                            {
                                "content": "            if self.credential and ui.yesno(",
                                "type": "added"
                            },
                            {
                                "content": "                    title=title,",
                                "type": "context"
                            },
                            {
                                "content": "                    text=\"Do you want to %s\" % action_msg):",
                                "type": "context"
                            },
                            {
                                "content": "                self.credential.enter_new()",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/downloaders/base.py",
                            "type": "method",
                            "name": "BaseDownloader._enter_credentials",
                            "content": "class BaseDownloader(object):\n    \"\"\"Base class for the downloaders\"\"\"\n\n    _DEFAULT_AUTHENTICATOR = None\n    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000\n\n    def __init__(self, credential=None, authenticator=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        credential: Credential, optional\n          Provides necessary credential fields to be used by authenticator\n        authenticator: Authenticator, optional\n          Authenticator to use for authentication.\n        \"\"\"\n        if not authenticator and self._DEFAULT_AUTHENTICATOR:\n            authenticator = self._DEFAULT_AUTHENTICATOR()\n\n        if authenticator and authenticator.requires_authentication:\n            if not credential and not authenticator.allows_anonymous:\n                msg = \"Both authenticator and credentials must be provided.\" \\\n                      \" Got only authenticator %s\" % repr(authenticator)\n                if ui.yesno(\n                    title=msg,\n                    text=\"Do you want to enter %s credentials to be used?\" % authenticator.DEFAULT_CREDENTIAL_TYPE\n                ):\n                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](\n                        \"session-only-for-%s\" % id(authenticator))\n                    credential.enter_new()\n                    # TODO: give an option to store those credentials, and generate a basic provider\n                    # record?\n                else:\n                    raise ValueError(msg)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._cache = None  # for fetches, not downloads\n\n    def access(self, method, url, allow_old_session=True, **kwargs):\n        \"\"\"Generic decorator to manage access to the URL via some method\n\n        Parameters\n        ----------\n        method : callable\n          A callable, usually a method of the same class, which we decorate\n          with access handling, and pass url as the first argument\n        url : string\n          URL to access\n        *args, **kwargs\n          Passed into the method call\n\n        Returns\n        -------\n        None or bytes\n        \"\"\"\n        # TODO: possibly wrap this logic outside within a decorator, which\n        # would just call the corresponding method\n        authenticator = self.authenticator\n        if authenticator:\n            needs_authentication = authenticator.requires_authentication\n        else:\n            needs_authentication = self.credential\n\n        attempt, incomplete_attempt = 0, 0\n        while True:\n            attempt += 1\n            if attempt > 20:\n                # are we stuck in a loop somehow? I think logic doesn't allow this atm\n                raise RuntimeError(\"Got to the %d'th iteration while trying to download %s\" % (attempt, url))\n            exc_info = None\n            supported_auth_types = []\n            try:\n                used_old_session = False\n                access_denied = False\n                used_old_session = self._establish_session(url, allow_old=allow_old_session)\n                if not allow_old_session:\n                    assert(not used_old_session)\n                lgr.log(5, \"Calling out into %s for %s\" % (method, url))\n                result = method(url, **kwargs)\n                # assume success if no puke etc\n                break\n            except AccessDeniedError as e:\n                if isinstance(e, AnonymousAccessDeniedError):\n                    access_denied = \"Anonymous\"\n                else:\n                    access_denied = \"Authenticated\"\n                lgr.debug(\"%s access was denied: %s\", access_denied, exc_str(e))\n                supported_auth_types = e.supported_types\n                exc_info = sys.exc_info()\n\n            except IncompleteDownloadError as e:\n                exc_info = sys.exc_info()\n                incomplete_attempt += 1\n                if incomplete_attempt > 5:\n                    # give up\n                    raise\n                lgr.debug(\"Failed to download fully, will try again: %s\", exc_str(e))\n                # TODO: may be fail ealier than after 20 attempts in such a case?\n            except DownloadError:\n                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!\n                raise\n\n            msg_types = ''\n            if supported_auth_types:\n                msg_types = \" The failure response indicated that following \" \\\n                            \"authentication types should be used: %s\" % (', '.join(supported_auth_types))\n            if access_denied:  # moved logic outside of except for clarity\n                # TODO: what if it was anonimous attempt without authentication,\n                #     so it is not \"requires_authentication\" but rather\n                #     \"supports_authentication\"?  We should not report below in\n                # _get_new_credential that authentication has failed then since there\n                # were no authentication.  We might need a custom exception to\n                # be caught above about that\n                if needs_authentication:\n                    # so we knew it needs authentication\n                    if used_old_session:\n                        # Let's try with fresh ones\n                        allow_old_session = False\n                        continue\n                    else:\n                        # we did use new cookies, we knew that authentication is needed\n                        # but still failed. So possible cases:\n                        #  1. authentication credentials changed/were revoked\n                        #     - allow user to re-enter credentials\n                        #  2. authentication mechanisms changed\n                        #     - we can't do anything here about that\n                        #  3. bug in out code which would render authentication/cookie handling\n                        #     ineffective\n                        #     - not sure what to do about it\n                        if not ui.is_interactive:\n                            lgr.error(\n                                \"Interface is non interactive, so we are \"\n                                \"reraising: %s\" % exc_str(e))\n                            reraise(*exc_info)\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=False)\n                        allow_old_session = False\n                        continue\n                else:  # None or False\n                    if needs_authentication is False:\n                        # those urls must or should NOT require authentication\n                        # but we got denied\n                        raise DownloadError(\n                            \"Failed to download from %s, which must be available\"\n                            \"without authentication but access was denied. \"\n                            \"Adjust your configuration for the provider.%s\"\n                            % (url, msg_types))\n                    else:\n                        # how could be None or any other non-False bool(False)\n                        assert(needs_authentication is None)\n                        # So we didn't know if authentication necessary, and it\n                        # seems to be necessary, so Let's ask the user to setup\n                        # authentication mechanism for this website\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=True)\n                        allow_old_session = False\n                        continue\n\n        return result\n\n    def _setup_new_provider(self, title, url, auth_types=None):\n        # Full new provider (TODO move into Providers?)\n        from .providers import Providers\n        providers = Providers.from_config_files()\n        while True:\n            provider = providers.enter_new(url, auth_types=auth_types)\n            if not provider:\n                if ui.yesno(\n                    title=\"Re-enter provider?\",\n                    text=\"You haven't entered or saved provider, would you like to retry?\",\n                    default=True\n                ):\n                    continue\n            break\n        return provider\n\n\n    def _enter_credentials(\n            self, url, denied_msg,\n            auth_types=[], new_provider=True):\n        \"\"\"Use when authentication fails to set new credentials for url\n\n        Raises\n        ------\n        DownloadError\n          If no known credentials type or user refuses to update\n        \"\"\"\n        title = \"{msg} access to {url} has failed.\".format(\n            msg=denied_msg, url=url)\n\n        if new_provider:\n            # No credential was known, we need to create an\n            # appropriate one\n            if not ui.yesno(\n                    title=title,\n                    text=\"Would you like to setup a new provider configuration\"\n                         \" to access url?\",\n                    default=True\n            ):\n                assert not self.authenticator, \"bug: incorrect assumption\"\n                raise DownloadError(\n                    title +\n                    \" No authenticator is known, cannot set any credential\")\n            else:\n                provider = self._setup_new_provider(\n                    title, url, auth_types=auth_types)\n                self.authenticator = provider.authenticator\n                self.credential = provider.credential\n                if not (self.credential and self.credential.is_known):\n                    # TODO: or should we ask to re-enter?\n                    self.credential.enter_new()\n        else:\n            action_msg = \"enter other credentials in case they were updated?\"\n\n            if ui.yesno(\n                    title=title,\n                    text=\"Do you want to %s\" % action_msg):\n                self.credential.enter_new()\n            else:\n                raise DownloadError(\n                    \"Failed to download from %s given available credentials\"\n                    % url)\n\n    @staticmethod\n    def _get_temp_download_filename(filepath):\n        \"\"\"Given a filepath, return the one to use as temp file during download\n        \"\"\"\n        # TODO: might better reside somewhere under .datalad/tmp or\n        # .git/datalad/tmp\n        return filepath + \".datalad-download-temp\"\n\n    @abstractmethod\n    def get_downloader_session(self, url):\n        \"\"\"\n\n        Parameters\n        ----------\n        url : str\n\n        Returns\n        -------\n        downloader_into_fp: callable\n           Which takes two parameters: file, pbar\n        target_size: int or None (if unknown)\n        url: str\n           Possibly redirected url\n        url_filename: str or None\n           Filename as decided from the (redirected) url\n        headers : dict or None\n        \"\"\"\n        raise NotImplementedError(\"Must be implemented in the subclass\")\n\n    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):\n        \"\"\"Verify that download finished correctly\"\"\"\n\n        if (self.authenticator\n                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \\\n                and hasattr(self.authenticator, 'failure_re') \\\n                and self.authenticator.failure_re:\n            assert hasattr(self.authenticator, 'check_for_auth_failure'), \\\n                \"%s has failure_re defined but no check_for_auth_failure\" \\\n                % self.authenticator\n\n            if file_:\n                with open(file_) as fp:\n                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)\n            else:\n                assert(content is not None)\n\n            self.authenticator.check_for_auth_failure(\n                content, \"Download of the url %s has failed: \" % url)\n\n        if target_size and target_size != downloaded_size:\n            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(\n                \"Downloaded size %d differs from originally announced %d\" % (downloaded_size, target_size))\n\n    def _download(self, url, path=None, overwrite=False, size=None, stats=None):\n        \"\"\"Download content into a file\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        path: str, optional\n          Path to file where to store the downloaded content.  If None,\n          filename deduced from the url and saved in curdir\n        size: int, optional\n          Limit in size to be downloaded\n\n        Returns\n        -------\n        None or string\n          Returns downloaded filename\n\n        \"\"\"\n\n        downloader_session = self.get_downloader_session(url)\n        status = self.get_status_from_headers(downloader_session.headers)\n\n        target_size = downloader_session.size\n        if size is not None:\n            target_size = min(target_size, size)\n\n        #### Specific to download\n        if path:\n            if isdir(path):\n                # provided path is a directory under which to save\n                filename = downloader_session.filename\n                if not filename:\n                    raise DownloadError(\n                        \"File name could not be determined from {}\".format(url))\n                filepath = opj(path, filename)\n            else:\n                filepath = path\n        else:\n            filepath = downloader_session.filename\n\n        existed = exists(filepath)\n        if existed and not overwrite:\n            raise DownloadError(\"File %s already exists\" % filepath)\n\n        # FETCH CONTENT\n        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])\n        try:\n            temp_filepath = self._get_temp_download_filename(filepath)\n            if exists(temp_filepath):\n                # eventually we might want to continue the download\n                lgr.warning(\n                    \"Temporary file %s from the previous download was found. \"\n                    \"It will be overriden\" % temp_filepath)\n                # TODO.  also logic below would clean it up atm\n\n            with open(temp_filepath, 'wb') as fp:\n                # TODO: url might be a bit too long for the beast.\n                # Consider to improve to make it animated as well, or shorten here\n                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n                t0 = time.time()\n                downloader_session.download(fp, pbar, size=size)\n                downloaded_time = time.time() - t0\n                pbar.finish()\n            downloaded_size = os.stat(temp_filepath).st_size\n\n            # (headers.get('Content-type', \"\") and headers.get('Content-Type')).startswith('text/html')\n            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator\n            self._verify_download(url, downloaded_size, target_size, temp_filepath)\n\n            # adjust atime/mtime according to headers/status\n            if status.mtime:\n                lgr.log(5, \"Setting mtime for %s to be %s\", temp_filepath, status.mtime)\n                os.utime(temp_filepath, (time.time(), status.mtime))\n\n            # place successfully downloaded over the filepath\n            os.rename(temp_filepath, filepath)\n\n            if stats:\n                stats.downloaded += 1\n                stats.overwritten += int(existed)\n                stats.downloaded_size += downloaded_size\n                stats.downloaded_time += downloaded_time\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to download {url} into {filepath}: {e_str}\".format(\n                **locals()\n            ))\n            raise DownloadError(exc_str(e))  # for now\n        finally:\n            if exists(temp_filepath):\n                # clean up\n                lgr.debug(\"Removing a temporary download %s\", temp_filepath)\n                unlink(temp_filepath)\n\n        return filepath\n\n    def download(self, url, path=None, **kwargs):\n        \"\"\"Fetch content as pointed by the URL optionally into a file\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        path : string, optional\n          Filename or existing directory to store downloaded content under.\n          If not provided -- deduced from the url\n\n        Returns\n        -------\n        string\n          file path\n        \"\"\"\n        # TODO: may be move all the path dealing logic here\n        # but then it might require sending request anyways for Content-Disposition\n        # so probably nah\n        lgr.info(\"Downloading %r into %r\", url, path)\n        return self.access(self._download, url, path=path, **kwargs)\n\n    @property\n    def cache(self):\n        if self._cache is None:\n            # TODO: move this all logic outside into a dedicated caching beast\n            lgr.info(\"Initializing cache for fetches\")\n            if PY2:\n                import anydbm as dbm\n            else:\n                import dbm\n            # Initiate cache.\n            # Very rudimentary caching for now, might fail many ways\n            cache_dir = cfg.obtain('datalad.locations.cache')\n            if not exists(cache_dir):\n                os.makedirs(cache_dir)\n            cache_path = opj(cache_dir, 'crawl_cache.dbm')\n            self._cache = dbm.open(cache_path, 'c')\n            import atexit\n            atexit.register(self._cache.close)\n        return self._cache\n\n    def _fetch(self, url, cache=None, size=None, allow_redirects=True):\n        \"\"\"Fetch content from a url into a file.\n\n        Very similar to _download but lacks any \"file\" management and decodes\n        content\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        cache: bool, optional\n          If None, config is consulted to determine whether results should be\n          cached. Cache is operating based on url, so no verification of any\n          kind is carried out\n\n        Returns\n        -------\n        bytes, dict\n          content, headers\n        \"\"\"\n        lgr.log(3, \"_fetch(%r, cache=%r, size=%r, allow_redirects=%r)\",\n                url, cache, size, allow_redirects)\n        if cache is None:\n            cache = cfg.obtain('datalad.crawl.cache', default=False)\n\n        if cache:\n            cache_key = msgpack.dumps(url)\n            lgr.debug(\"Loading content for url %s from cache\", url)\n            res = self.cache.get(cache_key)\n            if res is not None:\n                try:\n                    return msgpack.loads(res, encoding='utf-8')\n                except Exception as exc:\n                    lgr.warning(\"Failed to unpack loaded from cache for %s: %s\",\n                                url, exc_str(exc))\n\n        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)\n\n        target_size = downloader_session.size\n        if size is not None:\n            if size == 0:\n                # no download of the content was requested -- just return headers and be done\n                return None, downloader_session.headers\n            target_size = min(size, target_size)\n\n        # FETCH CONTENT\n        try:\n            # Consider to improve to make it animated as well, or shorten here\n            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n            content = downloader_session.download(size=size)\n            #pbar.finish()\n            downloaded_size = len(content)\n\n            # now that we know size based on encoded content, let's decode into string type\n            if PY3 and isinstance(content, binary_type):\n                content = content.decode()\n            # downloaded_size = os.stat(temp_filepath).st_size\n\n            self._verify_download(url, downloaded_size, target_size, None, content=content)\n\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to fetch {url}: {e_str}\".format(**locals()))\n            raise DownloadError(exc_str(e, limit=8))  # for now\n\n        if cache:\n            # apparently requests' CaseInsensitiveDict is not serialazable\n            # TODO:  may be we should reuse that type everywhere, to avoid\n            # out own handling for case-handling\n            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))\n\n        return content, downloader_session.headers\n\n    def fetch(self, url, **kwargs):\n        \"\"\"Fetch and return content (not decoded) as pointed by the URL\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        bytes\n          content\n        \"\"\"\n        lgr.info(\"Fetching %r\", url)\n        # Do not return headers, just content\n        out = self.access(self._fetch, url, **kwargs)\n        return out[0]\n\n    def get_status(self, url, old_status=None, **kwargs):\n        \"\"\"Return status of the url as a dict, None if N/A\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        old_status : FileStatus, optional\n          Previous status record.  If provided, might serve as a shortcut\n          to assess if status has changed, and if not -- return the same\n          record\n\n        Returns\n        -------\n        dict\n          dict-like beast depicting the status of the URL if accessible.\n          Returned value should be sufficient to tell if the URL content\n          has changed by comparing to previously obtained value.\n          If URL is not reachable, None would be returned\n        \"\"\"\n        return self.access(self._get_status, url, old_status=old_status, **kwargs)\n\n    # TODO: borrow from itself... ?\n    # @borrowkwargs(BaseDownloader, 'get_status')\n    def _get_status(self, url, old_status=None):\n\n        # the tricky part is only to make sure that we are getting the target URL\n        # and not some page saying to login, that is why we need to fetch some content\n        # in those cases, and not just check the headers\n        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \\\n            if self.authenticator \\\n            and hasattr(self.authenticator, 'failure_re') \\\n            and self.authenticator.failure_re \\\n            else 0\n\n        _, headers = self._fetch(url, cache=False, size=download_size)\n\n        # extract from headers information to depict the status of the url\n        status = self.get_status_from_headers(headers)\n\n        if old_status is not None:\n            raise NotImplementedError(\"Do not know yet how to deal with old_status. TODO\")\n\n        return status\n\n    @classmethod\n    @abstractmethod\n    def get_status_from_headers(cls, headers):\n        raise NotImplementedError(\"Implement in the subclass: %s\" % cls)\n\n    def get_target_url(self, url):\n        \"\"\"Return url after possible redirections\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return self.access(self._get_target_url, url)\n\n    def _get_target_url(self, url):\n        return self.get_downloader_session(url).url",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 250,
                            "end_lineno": 294,
                            "parent": {
                                "file_name": "datalad/downloaders/base.py",
                                "type": "class",
                                "name": "BaseDownloader",
                                "content": "class BaseDownloader(object):\n    \"\"\"Base class for the downloaders\"\"\"\n\n    _DEFAULT_AUTHENTICATOR = None\n    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000\n\n    def __init__(self, credential=None, authenticator=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        credential: Credential, optional\n          Provides necessary credential fields to be used by authenticator\n        authenticator: Authenticator, optional\n          Authenticator to use for authentication.\n        \"\"\"\n        if not authenticator and self._DEFAULT_AUTHENTICATOR:\n            authenticator = self._DEFAULT_AUTHENTICATOR()\n\n        if authenticator and authenticator.requires_authentication:\n            if not credential and not authenticator.allows_anonymous:\n                msg = \"Both authenticator and credentials must be provided.\" \\\n                      \" Got only authenticator %s\" % repr(authenticator)\n                if ui.yesno(\n                    title=msg,\n                    text=\"Do you want to enter %s credentials to be used?\" % authenticator.DEFAULT_CREDENTIAL_TYPE\n                ):\n                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](\n                        \"session-only-for-%s\" % id(authenticator))\n                    credential.enter_new()\n                    # TODO: give an option to store those credentials, and generate a basic provider\n                    # record?\n                else:\n                    raise ValueError(msg)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._cache = None  # for fetches, not downloads\n\n    def access(self, method, url, allow_old_session=True, **kwargs):\n        \"\"\"Generic decorator to manage access to the URL via some method\n\n        Parameters\n        ----------\n        method : callable\n          A callable, usually a method of the same class, which we decorate\n          with access handling, and pass url as the first argument\n        url : string\n          URL to access\n        *args, **kwargs\n          Passed into the method call\n\n        Returns\n        -------\n        None or bytes\n        \"\"\"\n        # TODO: possibly wrap this logic outside within a decorator, which\n        # would just call the corresponding method\n        authenticator = self.authenticator\n        if authenticator:\n            needs_authentication = authenticator.requires_authentication\n        else:\n            needs_authentication = self.credential\n\n        attempt, incomplete_attempt = 0, 0\n        while True:\n            attempt += 1\n            if attempt > 20:\n                # are we stuck in a loop somehow? I think logic doesn't allow this atm\n                raise RuntimeError(\"Got to the %d'th iteration while trying to download %s\" % (attempt, url))\n            exc_info = None\n            supported_auth_types = []\n            try:\n                used_old_session = False\n                access_denied = False\n                used_old_session = self._establish_session(url, allow_old=allow_old_session)\n                if not allow_old_session:\n                    assert(not used_old_session)\n                lgr.log(5, \"Calling out into %s for %s\" % (method, url))\n                result = method(url, **kwargs)\n                # assume success if no puke etc\n                break\n            except AccessDeniedError as e:\n                if isinstance(e, AnonymousAccessDeniedError):\n                    access_denied = \"Anonymous\"\n                else:\n                    access_denied = \"Authenticated\"\n                lgr.debug(\"%s access was denied: %s\", access_denied, exc_str(e))\n                supported_auth_types = e.supported_types\n                exc_info = sys.exc_info()\n\n            except IncompleteDownloadError as e:\n                exc_info = sys.exc_info()\n                incomplete_attempt += 1\n                if incomplete_attempt > 5:\n                    # give up\n                    raise\n                lgr.debug(\"Failed to download fully, will try again: %s\", exc_str(e))\n                # TODO: may be fail ealier than after 20 attempts in such a case?\n            except DownloadError:\n                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!\n                raise\n\n            msg_types = ''\n            if supported_auth_types:\n                msg_types = \" The failure response indicated that following \" \\\n                            \"authentication types should be used: %s\" % (', '.join(supported_auth_types))\n            if access_denied:  # moved logic outside of except for clarity\n                # TODO: what if it was anonimous attempt without authentication,\n                #     so it is not \"requires_authentication\" but rather\n                #     \"supports_authentication\"?  We should not report below in\n                # _get_new_credential that authentication has failed then since there\n                # were no authentication.  We might need a custom exception to\n                # be caught above about that\n                if needs_authentication:\n                    # so we knew it needs authentication\n                    if used_old_session:\n                        # Let's try with fresh ones\n                        allow_old_session = False\n                        continue\n                    else:\n                        # we did use new cookies, we knew that authentication is needed\n                        # but still failed. So possible cases:\n                        #  1. authentication credentials changed/were revoked\n                        #     - allow user to re-enter credentials\n                        #  2. authentication mechanisms changed\n                        #     - we can't do anything here about that\n                        #  3. bug in out code which would render authentication/cookie handling\n                        #     ineffective\n                        #     - not sure what to do about it\n                        if not ui.is_interactive:\n                            lgr.error(\n                                \"Interface is non interactive, so we are \"\n                                \"reraising: %s\" % exc_str(e))\n                            reraise(*exc_info)\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=False)\n                        allow_old_session = False\n                        continue\n                else:  # None or False\n                    if needs_authentication is False:\n                        # those urls must or should NOT require authentication\n                        # but we got denied\n                        raise DownloadError(\n                            \"Failed to download from %s, which must be available\"\n                            \"without authentication but access was denied. \"\n                            \"Adjust your configuration for the provider.%s\"\n                            % (url, msg_types))\n                    else:\n                        # how could be None or any other non-False bool(False)\n                        assert(needs_authentication is None)\n                        # So we didn't know if authentication necessary, and it\n                        # seems to be necessary, so Let's ask the user to setup\n                        # authentication mechanism for this website\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=True)\n                        allow_old_session = False\n                        continue\n\n        return result\n\n    def _setup_new_provider(self, title, url, auth_types=None):\n        # Full new provider (TODO move into Providers?)\n        from .providers import Providers\n        providers = Providers.from_config_files()\n        while True:\n            provider = providers.enter_new(url, auth_types=auth_types)\n            if not provider:\n                if ui.yesno(\n                    title=\"Re-enter provider?\",\n                    text=\"You haven't entered or saved provider, would you like to retry?\",\n                    default=True\n                ):\n                    continue\n            break\n        return provider\n\n\n    def _enter_credentials(\n            self, url, denied_msg,\n            auth_types=[], new_provider=True):\n        \"\"\"Use when authentication fails to set new credentials for url\n\n        Raises\n        ------\n        DownloadError\n          If no known credentials type or user refuses to update\n        \"\"\"\n        title = \"{msg} access to {url} has failed.\".format(\n            msg=denied_msg, url=url)\n\n        if new_provider:\n            # No credential was known, we need to create an\n            # appropriate one\n            if not ui.yesno(\n                    title=title,\n                    text=\"Would you like to setup a new provider configuration\"\n                         \" to access url?\",\n                    default=True\n            ):\n                assert not self.authenticator, \"bug: incorrect assumption\"\n                raise DownloadError(\n                    title +\n                    \" No authenticator is known, cannot set any credential\")\n            else:\n                provider = self._setup_new_provider(\n                    title, url, auth_types=auth_types)\n                self.authenticator = provider.authenticator\n                self.credential = provider.credential\n                if not (self.credential and self.credential.is_known):\n                    # TODO: or should we ask to re-enter?\n                    self.credential.enter_new()\n        else:\n            action_msg = \"enter other credentials in case they were updated?\"\n\n            if ui.yesno(\n                    title=title,\n                    text=\"Do you want to %s\" % action_msg):\n                self.credential.enter_new()\n            else:\n                raise DownloadError(\n                    \"Failed to download from %s given available credentials\"\n                    % url)\n\n    @staticmethod\n    def _get_temp_download_filename(filepath):\n        \"\"\"Given a filepath, return the one to use as temp file during download\n        \"\"\"\n        # TODO: might better reside somewhere under .datalad/tmp or\n        # .git/datalad/tmp\n        return filepath + \".datalad-download-temp\"\n\n    @abstractmethod\n    def get_downloader_session(self, url):\n        \"\"\"\n\n        Parameters\n        ----------\n        url : str\n\n        Returns\n        -------\n        downloader_into_fp: callable\n           Which takes two parameters: file, pbar\n        target_size: int or None (if unknown)\n        url: str\n           Possibly redirected url\n        url_filename: str or None\n           Filename as decided from the (redirected) url\n        headers : dict or None\n        \"\"\"\n        raise NotImplementedError(\"Must be implemented in the subclass\")\n\n    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):\n        \"\"\"Verify that download finished correctly\"\"\"\n\n        if (self.authenticator\n                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \\\n                and hasattr(self.authenticator, 'failure_re') \\\n                and self.authenticator.failure_re:\n            assert hasattr(self.authenticator, 'check_for_auth_failure'), \\\n                \"%s has failure_re defined but no check_for_auth_failure\" \\\n                % self.authenticator\n\n            if file_:\n                with open(file_) as fp:\n                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)\n            else:\n                assert(content is not None)\n\n            self.authenticator.check_for_auth_failure(\n                content, \"Download of the url %s has failed: \" % url)\n\n        if target_size and target_size != downloaded_size:\n            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(\n                \"Downloaded size %d differs from originally announced %d\" % (downloaded_size, target_size))\n\n    def _download(self, url, path=None, overwrite=False, size=None, stats=None):\n        \"\"\"Download content into a file\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        path: str, optional\n          Path to file where to store the downloaded content.  If None,\n          filename deduced from the url and saved in curdir\n        size: int, optional\n          Limit in size to be downloaded\n\n        Returns\n        -------\n        None or string\n          Returns downloaded filename\n\n        \"\"\"\n\n        downloader_session = self.get_downloader_session(url)\n        status = self.get_status_from_headers(downloader_session.headers)\n\n        target_size = downloader_session.size\n        if size is not None:\n            target_size = min(target_size, size)\n\n        #### Specific to download\n        if path:\n            if isdir(path):\n                # provided path is a directory under which to save\n                filename = downloader_session.filename\n                if not filename:\n                    raise DownloadError(\n                        \"File name could not be determined from {}\".format(url))\n                filepath = opj(path, filename)\n            else:\n                filepath = path\n        else:\n            filepath = downloader_session.filename\n\n        existed = exists(filepath)\n        if existed and not overwrite:\n            raise DownloadError(\"File %s already exists\" % filepath)\n\n        # FETCH CONTENT\n        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])\n        try:\n            temp_filepath = self._get_temp_download_filename(filepath)\n            if exists(temp_filepath):\n                # eventually we might want to continue the download\n                lgr.warning(\n                    \"Temporary file %s from the previous download was found. \"\n                    \"It will be overriden\" % temp_filepath)\n                # TODO.  also logic below would clean it up atm\n\n            with open(temp_filepath, 'wb') as fp:\n                # TODO: url might be a bit too long for the beast.\n                # Consider to improve to make it animated as well, or shorten here\n                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n                t0 = time.time()\n                downloader_session.download(fp, pbar, size=size)\n                downloaded_time = time.time() - t0\n                pbar.finish()\n            downloaded_size = os.stat(temp_filepath).st_size\n\n            # (headers.get('Content-type', \"\") and headers.get('Content-Type')).startswith('text/html')\n            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator\n            self._verify_download(url, downloaded_size, target_size, temp_filepath)\n\n            # adjust atime/mtime according to headers/status\n            if status.mtime:\n                lgr.log(5, \"Setting mtime for %s to be %s\", temp_filepath, status.mtime)\n                os.utime(temp_filepath, (time.time(), status.mtime))\n\n            # place successfully downloaded over the filepath\n            os.rename(temp_filepath, filepath)\n\n            if stats:\n                stats.downloaded += 1\n                stats.overwritten += int(existed)\n                stats.downloaded_size += downloaded_size\n                stats.downloaded_time += downloaded_time\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to download {url} into {filepath}: {e_str}\".format(\n                **locals()\n            ))\n            raise DownloadError(exc_str(e))  # for now\n        finally:\n            if exists(temp_filepath):\n                # clean up\n                lgr.debug(\"Removing a temporary download %s\", temp_filepath)\n                unlink(temp_filepath)\n\n        return filepath\n\n    def download(self, url, path=None, **kwargs):\n        \"\"\"Fetch content as pointed by the URL optionally into a file\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        path : string, optional\n          Filename or existing directory to store downloaded content under.\n          If not provided -- deduced from the url\n\n        Returns\n        -------\n        string\n          file path\n        \"\"\"\n        # TODO: may be move all the path dealing logic here\n        # but then it might require sending request anyways for Content-Disposition\n        # so probably nah\n        lgr.info(\"Downloading %r into %r\", url, path)\n        return self.access(self._download, url, path=path, **kwargs)\n\n    @property\n    def cache(self):\n        if self._cache is None:\n            # TODO: move this all logic outside into a dedicated caching beast\n            lgr.info(\"Initializing cache for fetches\")\n            if PY2:\n                import anydbm as dbm\n            else:\n                import dbm\n            # Initiate cache.\n            # Very rudimentary caching for now, might fail many ways\n            cache_dir = cfg.obtain('datalad.locations.cache')\n            if not exists(cache_dir):\n                os.makedirs(cache_dir)\n            cache_path = opj(cache_dir, 'crawl_cache.dbm')\n            self._cache = dbm.open(cache_path, 'c')\n            import atexit\n            atexit.register(self._cache.close)\n        return self._cache\n\n    def _fetch(self, url, cache=None, size=None, allow_redirects=True):\n        \"\"\"Fetch content from a url into a file.\n\n        Very similar to _download but lacks any \"file\" management and decodes\n        content\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        cache: bool, optional\n          If None, config is consulted to determine whether results should be\n          cached. Cache is operating based on url, so no verification of any\n          kind is carried out\n\n        Returns\n        -------\n        bytes, dict\n          content, headers\n        \"\"\"\n        lgr.log(3, \"_fetch(%r, cache=%r, size=%r, allow_redirects=%r)\",\n                url, cache, size, allow_redirects)\n        if cache is None:\n            cache = cfg.obtain('datalad.crawl.cache', default=False)\n\n        if cache:\n            cache_key = msgpack.dumps(url)\n            lgr.debug(\"Loading content for url %s from cache\", url)\n            res = self.cache.get(cache_key)\n            if res is not None:\n                try:\n                    return msgpack.loads(res, encoding='utf-8')\n                except Exception as exc:\n                    lgr.warning(\"Failed to unpack loaded from cache for %s: %s\",\n                                url, exc_str(exc))\n\n        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)\n\n        target_size = downloader_session.size\n        if size is not None:\n            if size == 0:\n                # no download of the content was requested -- just return headers and be done\n                return None, downloader_session.headers\n            target_size = min(size, target_size)\n\n        # FETCH CONTENT\n        try:\n            # Consider to improve to make it animated as well, or shorten here\n            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n            content = downloader_session.download(size=size)\n            #pbar.finish()\n            downloaded_size = len(content)\n\n            # now that we know size based on encoded content, let's decode into string type\n            if PY3 and isinstance(content, binary_type):\n                content = content.decode()\n            # downloaded_size = os.stat(temp_filepath).st_size\n\n            self._verify_download(url, downloaded_size, target_size, None, content=content)\n\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to fetch {url}: {e_str}\".format(**locals()))\n            raise DownloadError(exc_str(e, limit=8))  # for now\n\n        if cache:\n            # apparently requests' CaseInsensitiveDict is not serialazable\n            # TODO:  may be we should reuse that type everywhere, to avoid\n            # out own handling for case-handling\n            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))\n\n        return content, downloader_session.headers\n\n    def fetch(self, url, **kwargs):\n        \"\"\"Fetch and return content (not decoded) as pointed by the URL\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        bytes\n          content\n        \"\"\"\n        lgr.info(\"Fetching %r\", url)\n        # Do not return headers, just content\n        out = self.access(self._fetch, url, **kwargs)\n        return out[0]\n\n    def get_status(self, url, old_status=None, **kwargs):\n        \"\"\"Return status of the url as a dict, None if N/A\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        old_status : FileStatus, optional\n          Previous status record.  If provided, might serve as a shortcut\n          to assess if status has changed, and if not -- return the same\n          record\n\n        Returns\n        -------\n        dict\n          dict-like beast depicting the status of the URL if accessible.\n          Returned value should be sufficient to tell if the URL content\n          has changed by comparing to previously obtained value.\n          If URL is not reachable, None would be returned\n        \"\"\"\n        return self.access(self._get_status, url, old_status=old_status, **kwargs)\n\n    # TODO: borrow from itself... ?\n    # @borrowkwargs(BaseDownloader, 'get_status')\n    def _get_status(self, url, old_status=None):\n\n        # the tricky part is only to make sure that we are getting the target URL\n        # and not some page saying to login, that is why we need to fetch some content\n        # in those cases, and not just check the headers\n        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \\\n            if self.authenticator \\\n            and hasattr(self.authenticator, 'failure_re') \\\n            and self.authenticator.failure_re \\\n            else 0\n\n        _, headers = self._fetch(url, cache=False, size=download_size)\n\n        # extract from headers information to depict the status of the url\n        status = self.get_status_from_headers(headers)\n\n        if old_status is not None:\n            raise NotImplementedError(\"Do not know yet how to deal with old_status. TODO\")\n\n        return status\n\n    @classmethod\n    @abstractmethod\n    def get_status_from_headers(cls, headers):\n        raise NotImplementedError(\"Implement in the subclass: %s\" % cls)\n\n    def get_target_url(self, url):\n        \"\"\"Return url after possible redirections\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return self.access(self._get_target_url, url)\n\n    def _get_target_url(self, url):\n        return self.get_downloader_session(url).url",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 67,
                                "end_lineno": 648,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/downloaders/base.py",
                            "type": "class",
                            "name": "BaseDownloader",
                            "content": "class BaseDownloader(object):\n    \"\"\"Base class for the downloaders\"\"\"\n\n    _DEFAULT_AUTHENTICATOR = None\n    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000\n\n    def __init__(self, credential=None, authenticator=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        credential: Credential, optional\n          Provides necessary credential fields to be used by authenticator\n        authenticator: Authenticator, optional\n          Authenticator to use for authentication.\n        \"\"\"\n        if not authenticator and self._DEFAULT_AUTHENTICATOR:\n            authenticator = self._DEFAULT_AUTHENTICATOR()\n\n        if authenticator and authenticator.requires_authentication:\n            if not credential and not authenticator.allows_anonymous:\n                msg = \"Both authenticator and credentials must be provided.\" \\\n                      \" Got only authenticator %s\" % repr(authenticator)\n                if ui.yesno(\n                    title=msg,\n                    text=\"Do you want to enter %s credentials to be used?\" % authenticator.DEFAULT_CREDENTIAL_TYPE\n                ):\n                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](\n                        \"session-only-for-%s\" % id(authenticator))\n                    credential.enter_new()\n                    # TODO: give an option to store those credentials, and generate a basic provider\n                    # record?\n                else:\n                    raise ValueError(msg)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._cache = None  # for fetches, not downloads\n\n    def access(self, method, url, allow_old_session=True, **kwargs):\n        \"\"\"Generic decorator to manage access to the URL via some method\n\n        Parameters\n        ----------\n        method : callable\n          A callable, usually a method of the same class, which we decorate\n          with access handling, and pass url as the first argument\n        url : string\n          URL to access\n        *args, **kwargs\n          Passed into the method call\n\n        Returns\n        -------\n        None or bytes\n        \"\"\"\n        # TODO: possibly wrap this logic outside within a decorator, which\n        # would just call the corresponding method\n        authenticator = self.authenticator\n        if authenticator:\n            needs_authentication = authenticator.requires_authentication\n        else:\n            needs_authentication = self.credential\n\n        attempt, incomplete_attempt = 0, 0\n        while True:\n            attempt += 1\n            if attempt > 20:\n                # are we stuck in a loop somehow? I think logic doesn't allow this atm\n                raise RuntimeError(\"Got to the %d'th iteration while trying to download %s\" % (attempt, url))\n            exc_info = None\n            supported_auth_types = []\n            try:\n                used_old_session = False\n                access_denied = False\n                used_old_session = self._establish_session(url, allow_old=allow_old_session)\n                if not allow_old_session:\n                    assert(not used_old_session)\n                lgr.log(5, \"Calling out into %s for %s\" % (method, url))\n                result = method(url, **kwargs)\n                # assume success if no puke etc\n                break\n            except AccessDeniedError as e:\n                if isinstance(e, AnonymousAccessDeniedError):\n                    access_denied = \"Anonymous\"\n                else:\n                    access_denied = \"Authenticated\"\n                lgr.debug(\"%s access was denied: %s\", access_denied, exc_str(e))\n                supported_auth_types = e.supported_types\n                exc_info = sys.exc_info()\n\n            except IncompleteDownloadError as e:\n                exc_info = sys.exc_info()\n                incomplete_attempt += 1\n                if incomplete_attempt > 5:\n                    # give up\n                    raise\n                lgr.debug(\"Failed to download fully, will try again: %s\", exc_str(e))\n                # TODO: may be fail ealier than after 20 attempts in such a case?\n            except DownloadError:\n                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!\n                raise\n\n            msg_types = ''\n            if supported_auth_types:\n                msg_types = \" The failure response indicated that following \" \\\n                            \"authentication types should be used: %s\" % (', '.join(supported_auth_types))\n            if access_denied:  # moved logic outside of except for clarity\n                # TODO: what if it was anonimous attempt without authentication,\n                #     so it is not \"requires_authentication\" but rather\n                #     \"supports_authentication\"?  We should not report below in\n                # _get_new_credential that authentication has failed then since there\n                # were no authentication.  We might need a custom exception to\n                # be caught above about that\n                if needs_authentication:\n                    # so we knew it needs authentication\n                    if used_old_session:\n                        # Let's try with fresh ones\n                        allow_old_session = False\n                        continue\n                    else:\n                        # we did use new cookies, we knew that authentication is needed\n                        # but still failed. So possible cases:\n                        #  1. authentication credentials changed/were revoked\n                        #     - allow user to re-enter credentials\n                        #  2. authentication mechanisms changed\n                        #     - we can't do anything here about that\n                        #  3. bug in out code which would render authentication/cookie handling\n                        #     ineffective\n                        #     - not sure what to do about it\n                        if not ui.is_interactive:\n                            lgr.error(\n                                \"Interface is non interactive, so we are \"\n                                \"reraising: %s\" % exc_str(e))\n                            reraise(*exc_info)\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=False)\n                        allow_old_session = False\n                        continue\n                else:  # None or False\n                    if needs_authentication is False:\n                        # those urls must or should NOT require authentication\n                        # but we got denied\n                        raise DownloadError(\n                            \"Failed to download from %s, which must be available\"\n                            \"without authentication but access was denied. \"\n                            \"Adjust your configuration for the provider.%s\"\n                            % (url, msg_types))\n                    else:\n                        # how could be None or any other non-False bool(False)\n                        assert(needs_authentication is None)\n                        # So we didn't know if authentication necessary, and it\n                        # seems to be necessary, so Let's ask the user to setup\n                        # authentication mechanism for this website\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=True)\n                        allow_old_session = False\n                        continue\n\n        return result\n\n    def _setup_new_provider(self, title, url, auth_types=None):\n        # Full new provider (TODO move into Providers?)\n        from .providers import Providers\n        providers = Providers.from_config_files()\n        while True:\n            provider = providers.enter_new(url, auth_types=auth_types)\n            if not provider:\n                if ui.yesno(\n                    title=\"Re-enter provider?\",\n                    text=\"You haven't entered or saved provider, would you like to retry?\",\n                    default=True\n                ):\n                    continue\n            break\n        return provider\n\n\n    def _enter_credentials(\n            self, url, denied_msg,\n            auth_types=[], new_provider=True):\n        \"\"\"Use when authentication fails to set new credentials for url\n\n        Raises\n        ------\n        DownloadError\n          If no known credentials type or user refuses to update\n        \"\"\"\n        title = \"{msg} access to {url} has failed.\".format(\n            msg=denied_msg, url=url)\n\n        if new_provider:\n            # No credential was known, we need to create an\n            # appropriate one\n            if not ui.yesno(\n                    title=title,\n                    text=\"Would you like to setup a new provider configuration\"\n                         \" to access url?\",\n                    default=True\n            ):\n                assert not self.authenticator, \"bug: incorrect assumption\"\n                raise DownloadError(\n                    title +\n                    \" No authenticator is known, cannot set any credential\")\n            else:\n                provider = self._setup_new_provider(\n                    title, url, auth_types=auth_types)\n                self.authenticator = provider.authenticator\n                self.credential = provider.credential\n                if not (self.credential and self.credential.is_known):\n                    # TODO: or should we ask to re-enter?\n                    self.credential.enter_new()\n        else:\n            action_msg = \"enter other credentials in case they were updated?\"\n\n            if ui.yesno(\n                    title=title,\n                    text=\"Do you want to %s\" % action_msg):\n                self.credential.enter_new()\n            else:\n                raise DownloadError(\n                    \"Failed to download from %s given available credentials\"\n                    % url)\n\n    @staticmethod\n    def _get_temp_download_filename(filepath):\n        \"\"\"Given a filepath, return the one to use as temp file during download\n        \"\"\"\n        # TODO: might better reside somewhere under .datalad/tmp or\n        # .git/datalad/tmp\n        return filepath + \".datalad-download-temp\"\n\n    @abstractmethod\n    def get_downloader_session(self, url):\n        \"\"\"\n\n        Parameters\n        ----------\n        url : str\n\n        Returns\n        -------\n        downloader_into_fp: callable\n           Which takes two parameters: file, pbar\n        target_size: int or None (if unknown)\n        url: str\n           Possibly redirected url\n        url_filename: str or None\n           Filename as decided from the (redirected) url\n        headers : dict or None\n        \"\"\"\n        raise NotImplementedError(\"Must be implemented in the subclass\")\n\n    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):\n        \"\"\"Verify that download finished correctly\"\"\"\n\n        if (self.authenticator\n                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \\\n                and hasattr(self.authenticator, 'failure_re') \\\n                and self.authenticator.failure_re:\n            assert hasattr(self.authenticator, 'check_for_auth_failure'), \\\n                \"%s has failure_re defined but no check_for_auth_failure\" \\\n                % self.authenticator\n\n            if file_:\n                with open(file_) as fp:\n                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)\n            else:\n                assert(content is not None)\n\n            self.authenticator.check_for_auth_failure(\n                content, \"Download of the url %s has failed: \" % url)\n\n        if target_size and target_size != downloaded_size:\n            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(\n                \"Downloaded size %d differs from originally announced %d\" % (downloaded_size, target_size))\n\n    def _download(self, url, path=None, overwrite=False, size=None, stats=None):\n        \"\"\"Download content into a file\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        path: str, optional\n          Path to file where to store the downloaded content.  If None,\n          filename deduced from the url and saved in curdir\n        size: int, optional\n          Limit in size to be downloaded\n\n        Returns\n        -------\n        None or string\n          Returns downloaded filename\n\n        \"\"\"\n\n        downloader_session = self.get_downloader_session(url)\n        status = self.get_status_from_headers(downloader_session.headers)\n\n        target_size = downloader_session.size\n        if size is not None:\n            target_size = min(target_size, size)\n\n        #### Specific to download\n        if path:\n            if isdir(path):\n                # provided path is a directory under which to save\n                filename = downloader_session.filename\n                if not filename:\n                    raise DownloadError(\n                        \"File name could not be determined from {}\".format(url))\n                filepath = opj(path, filename)\n            else:\n                filepath = path\n        else:\n            filepath = downloader_session.filename\n\n        existed = exists(filepath)\n        if existed and not overwrite:\n            raise DownloadError(\"File %s already exists\" % filepath)\n\n        # FETCH CONTENT\n        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])\n        try:\n            temp_filepath = self._get_temp_download_filename(filepath)\n            if exists(temp_filepath):\n                # eventually we might want to continue the download\n                lgr.warning(\n                    \"Temporary file %s from the previous download was found. \"\n                    \"It will be overriden\" % temp_filepath)\n                # TODO.  also logic below would clean it up atm\n\n            with open(temp_filepath, 'wb') as fp:\n                # TODO: url might be a bit too long for the beast.\n                # Consider to improve to make it animated as well, or shorten here\n                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n                t0 = time.time()\n                downloader_session.download(fp, pbar, size=size)\n                downloaded_time = time.time() - t0\n                pbar.finish()\n            downloaded_size = os.stat(temp_filepath).st_size\n\n            # (headers.get('Content-type', \"\") and headers.get('Content-Type')).startswith('text/html')\n            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator\n            self._verify_download(url, downloaded_size, target_size, temp_filepath)\n\n            # adjust atime/mtime according to headers/status\n            if status.mtime:\n                lgr.log(5, \"Setting mtime for %s to be %s\", temp_filepath, status.mtime)\n                os.utime(temp_filepath, (time.time(), status.mtime))\n\n            # place successfully downloaded over the filepath\n            os.rename(temp_filepath, filepath)\n\n            if stats:\n                stats.downloaded += 1\n                stats.overwritten += int(existed)\n                stats.downloaded_size += downloaded_size\n                stats.downloaded_time += downloaded_time\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to download {url} into {filepath}: {e_str}\".format(\n                **locals()\n            ))\n            raise DownloadError(exc_str(e))  # for now\n        finally:\n            if exists(temp_filepath):\n                # clean up\n                lgr.debug(\"Removing a temporary download %s\", temp_filepath)\n                unlink(temp_filepath)\n\n        return filepath\n\n    def download(self, url, path=None, **kwargs):\n        \"\"\"Fetch content as pointed by the URL optionally into a file\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        path : string, optional\n          Filename or existing directory to store downloaded content under.\n          If not provided -- deduced from the url\n\n        Returns\n        -------\n        string\n          file path\n        \"\"\"\n        # TODO: may be move all the path dealing logic here\n        # but then it might require sending request anyways for Content-Disposition\n        # so probably nah\n        lgr.info(\"Downloading %r into %r\", url, path)\n        return self.access(self._download, url, path=path, **kwargs)\n\n    @property\n    def cache(self):\n        if self._cache is None:\n            # TODO: move this all logic outside into a dedicated caching beast\n            lgr.info(\"Initializing cache for fetches\")\n            if PY2:\n                import anydbm as dbm\n            else:\n                import dbm\n            # Initiate cache.\n            # Very rudimentary caching for now, might fail many ways\n            cache_dir = cfg.obtain('datalad.locations.cache')\n            if not exists(cache_dir):\n                os.makedirs(cache_dir)\n            cache_path = opj(cache_dir, 'crawl_cache.dbm')\n            self._cache = dbm.open(cache_path, 'c')\n            import atexit\n            atexit.register(self._cache.close)\n        return self._cache\n\n    def _fetch(self, url, cache=None, size=None, allow_redirects=True):\n        \"\"\"Fetch content from a url into a file.\n\n        Very similar to _download but lacks any \"file\" management and decodes\n        content\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        cache: bool, optional\n          If None, config is consulted to determine whether results should be\n          cached. Cache is operating based on url, so no verification of any\n          kind is carried out\n\n        Returns\n        -------\n        bytes, dict\n          content, headers\n        \"\"\"\n        lgr.log(3, \"_fetch(%r, cache=%r, size=%r, allow_redirects=%r)\",\n                url, cache, size, allow_redirects)\n        if cache is None:\n            cache = cfg.obtain('datalad.crawl.cache', default=False)\n\n        if cache:\n            cache_key = msgpack.dumps(url)\n            lgr.debug(\"Loading content for url %s from cache\", url)\n            res = self.cache.get(cache_key)\n            if res is not None:\n                try:\n                    return msgpack.loads(res, encoding='utf-8')\n                except Exception as exc:\n                    lgr.warning(\"Failed to unpack loaded from cache for %s: %s\",\n                                url, exc_str(exc))\n\n        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)\n\n        target_size = downloader_session.size\n        if size is not None:\n            if size == 0:\n                # no download of the content was requested -- just return headers and be done\n                return None, downloader_session.headers\n            target_size = min(size, target_size)\n\n        # FETCH CONTENT\n        try:\n            # Consider to improve to make it animated as well, or shorten here\n            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n            content = downloader_session.download(size=size)\n            #pbar.finish()\n            downloaded_size = len(content)\n\n            # now that we know size based on encoded content, let's decode into string type\n            if PY3 and isinstance(content, binary_type):\n                content = content.decode()\n            # downloaded_size = os.stat(temp_filepath).st_size\n\n            self._verify_download(url, downloaded_size, target_size, None, content=content)\n\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to fetch {url}: {e_str}\".format(**locals()))\n            raise DownloadError(exc_str(e, limit=8))  # for now\n\n        if cache:\n            # apparently requests' CaseInsensitiveDict is not serialazable\n            # TODO:  may be we should reuse that type everywhere, to avoid\n            # out own handling for case-handling\n            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))\n\n        return content, downloader_session.headers\n\n    def fetch(self, url, **kwargs):\n        \"\"\"Fetch and return content (not decoded) as pointed by the URL\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        bytes\n          content\n        \"\"\"\n        lgr.info(\"Fetching %r\", url)\n        # Do not return headers, just content\n        out = self.access(self._fetch, url, **kwargs)\n        return out[0]\n\n    def get_status(self, url, old_status=None, **kwargs):\n        \"\"\"Return status of the url as a dict, None if N/A\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        old_status : FileStatus, optional\n          Previous status record.  If provided, might serve as a shortcut\n          to assess if status has changed, and if not -- return the same\n          record\n\n        Returns\n        -------\n        dict\n          dict-like beast depicting the status of the URL if accessible.\n          Returned value should be sufficient to tell if the URL content\n          has changed by comparing to previously obtained value.\n          If URL is not reachable, None would be returned\n        \"\"\"\n        return self.access(self._get_status, url, old_status=old_status, **kwargs)\n\n    # TODO: borrow from itself... ?\n    # @borrowkwargs(BaseDownloader, 'get_status')\n    def _get_status(self, url, old_status=None):\n\n        # the tricky part is only to make sure that we are getting the target URL\n        # and not some page saying to login, that is why we need to fetch some content\n        # in those cases, and not just check the headers\n        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \\\n            if self.authenticator \\\n            and hasattr(self.authenticator, 'failure_re') \\\n            and self.authenticator.failure_re \\\n            else 0\n\n        _, headers = self._fetch(url, cache=False, size=download_size)\n\n        # extract from headers information to depict the status of the url\n        status = self.get_status_from_headers(headers)\n\n        if old_status is not None:\n            raise NotImplementedError(\"Do not know yet how to deal with old_status. TODO\")\n\n        return status\n\n    @classmethod\n    @abstractmethod\n    def get_status_from_headers(cls, headers):\n        raise NotImplementedError(\"Implement in the subclass: %s\" % cls)\n\n    def get_target_url(self, url):\n        \"\"\"Return url after possible redirections\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return self.access(self._get_target_url, url)\n\n    def _get_target_url(self, url):\n        return self.get_downloader_session(url).url",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 67,
                            "end_lineno": 648,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for S3 downloader\"\"\"\n\nimport os\nfrom mock import patch\n\nfrom ..s3 import S3Authenticator\nfrom ..providers import Providers  # to test against crcns\n\nfrom ...tests.utils import swallow_outputs\nfrom ...tests.utils import SkipTest\nfrom ...tests.utils import with_tempfile\nfrom ...tests.utils import assert_equal\nfrom ...tests.utils import assert_in\nfrom ...tests.utils import use_cassette\nfrom ...tests.utils import assert_raises\nfrom ...tests.utils import skip_if_no_network\nfrom ...dochelpers import exc_str\nfrom ...downloaders.base import DownloadError\n\ntry:\n    import boto\nexcept Exception as e:\n    raise SkipTest(\"boto module is not available: %s\" % exc_str(e))\n\nfrom .utils import get_test_providers\nfrom .test_http import check_download_external_url\n\nskip_if_no_network()  # TODO: provide persistent vcr fixtures for the tests\n\nurl_2versions_nonversioned1 = 's3://datalad-test0-versioned/2versions-nonversioned1.txt'\nurl_2versions_nonversioned1_ver1 = url_2versions_nonversioned1 + '?versionId=null'\nurl_2versions_nonversioned1_ver2 = url_2versions_nonversioned1 + '?versionId=V4Dqhu0QTEtxmvoNkCHGrjVZVomR1Ryo'\nurl_1version_bucketwithdot = 's3://datalad.test1/version1.txt'\n\n@use_cassette('test_s3_download_basic')\ndef test_s3_download_basic():\n\n    for url, success_str, failed_str in [\n        (url_2versions_nonversioned1, 'version2', 'version1'),\n        (url_2versions_nonversioned1_ver2, 'version2', 'version1'),\n        (url_2versions_nonversioned1_ver1, 'version1', 'version2'),\n        (url_1version_bucketwithdot, 'version1', 'nothing')\n    ]:\n        yield check_download_external_url, url, failed_str, success_str\n\n# TODO: redo smart way with mocking, to avoid unnecessary CPU waste\n@use_cassette('test_s3_mtime')\n@with_tempfile\ndef test_mtime(tempfile):\n    url = url_2versions_nonversioned1_ver2\n    with swallow_outputs():\n        # without allow_old=False it might be reusing previous connection\n        # which had already vcr tape for it, leading to failure.\n        # TODO:  make allow_old configurable and then within tests disallow\n        # allow_old\n        get_test_providers(url).download(url, path=tempfile, allow_old_session=False)\n    assert_equal(os.stat(tempfile).st_mtime, 1446873817)\n\n    # and if url is wrong\n    url = 's3://datalad-test0-versioned/nonexisting'\n    assert_raises(DownloadError, get_test_providers(url).download, url, path=tempfile, overwrite=True)\n\n\n@use_cassette('test_s3_reuse_session')\n@with_tempfile\n# forgot how to tell it not to change return value, so this side_effect beast now\n@patch.object(S3Authenticator, 'authenticate', side_effect=S3Authenticator.authenticate, autospec=True)\ndef test_reuse_session(tempfile, mocked_auth):\n    Providers.reset_default_providers()  # necessary for the testing below\n    providers = get_test_providers(url_2versions_nonversioned1_ver1)  # to check credentials\n    with swallow_outputs():\n        providers.download(url_2versions_nonversioned1_ver1, path=tempfile)\n    assert_equal(mocked_auth.call_count, 1)\n\n    providers2 = Providers.from_config_files()\n    with swallow_outputs():\n        providers2.download(url_2versions_nonversioned1_ver2, path=tempfile, overwrite=True)\n    assert_equal(mocked_auth.call_count, 1)\n\n    # but if we reload -- everything reloads and we need to authenticate again\n    providers2 = Providers.from_config_files(reload=True)\n    with swallow_outputs():\n        providers2.download(url_2versions_nonversioned1_ver2, path=tempfile, overwrite=True)\n    assert_equal(mocked_auth.call_count, 2)\n\n    Providers.reset_default_providers()  # necessary to avoid side-effects from having a vcr'ed connection\n    # leaking through default provider's bucket, e.g. breaking test_mtime if ran after this one\n\n\ndef test_parse_url():\n    from ..s3 import S3Downloader\n    f = S3Downloader._parse_url\n    b1 = \"s3://bucket.name/file/path?revision=123\"\n    assert_equal(f(b1, bucket_only=True), 'bucket.name')\n    assert_equal(f(b1), ('bucket.name', 'file/path', {'revision': '123'}))\n    assert_equal(f(\"s3://b/f name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%20name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%2Bname\"), ('b', 'f+name', {}))\n    assert_equal(f(\"s3://b/f%2bname?r=%20\"), ('b', 'f+name', {'r': '%20'}))",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for S3 downloader\"\"\"\n\nimport os\nfrom mock import patch\n\nfrom ..s3 import S3Authenticator\nfrom ..s3 import S3Downloader\nfrom ..providers import Providers  # to test against crcns\n\nfrom ...tests.utils import swallow_outputs\nfrom ...tests.utils import SkipTest\nfrom ...tests.utils import with_tempfile\nfrom ...tests.utils import assert_equal\nfrom ...tests.utils import assert_in\nfrom ...tests.utils import use_cassette\nfrom ...tests.utils import assert_raises\nfrom ...tests.utils import skip_if_no_network\nfrom ...tests.utils import with_testsui\nfrom ...dochelpers import exc_str\nfrom ...downloaders.base import DownloadError\nfrom ...support.exceptions import AccessDeniedError\n\ntry:\n    import boto\nexcept Exception as e:\n    raise SkipTest(\"boto module is not available: %s\" % exc_str(e))\n\nfrom .utils import get_test_providers\nfrom .test_http import check_download_external_url\n\nskip_if_no_network()  # TODO: provide persistent vcr fixtures for the tests\n\nurl_2versions_nonversioned1 = 's3://datalad-test0-versioned/2versions-nonversioned1.txt'\nurl_2versions_nonversioned1_ver1 = url_2versions_nonversioned1 + '?versionId=null'\nurl_2versions_nonversioned1_ver2 = url_2versions_nonversioned1 + '?versionId=V4Dqhu0QTEtxmvoNkCHGrjVZVomR1Ryo'\nurl_1version_bucketwithdot = 's3://datalad.test1/version1.txt'\n\n@use_cassette('test_s3_download_basic')\ndef test_s3_download_basic():\n\n    for url, success_str, failed_str in [\n        (url_2versions_nonversioned1, 'version2', 'version1'),\n        (url_2versions_nonversioned1_ver2, 'version2', 'version1'),\n        (url_2versions_nonversioned1_ver1, 'version1', 'version2'),\n        (url_1version_bucketwithdot, 'version1', 'nothing')\n    ]:\n        yield check_download_external_url, url, failed_str, success_str\n\n# TODO: redo smart way with mocking, to avoid unnecessary CPU waste\n@use_cassette('test_s3_mtime')\n@with_tempfile\ndef test_mtime(tempfile):\n    url = url_2versions_nonversioned1_ver2\n    with swallow_outputs():\n        # without allow_old=False it might be reusing previous connection\n        # which had already vcr tape for it, leading to failure.\n        # TODO:  make allow_old configurable and then within tests disallow\n        # allow_old\n        get_test_providers(url).download(url, path=tempfile, allow_old_session=False)\n    assert_equal(os.stat(tempfile).st_mtime, 1446873817)\n\n    # and if url is wrong\n    url = 's3://datalad-test0-versioned/nonexisting'\n    assert_raises(DownloadError, get_test_providers(url).download, url, path=tempfile, overwrite=True)\n\n\n@use_cassette('test_s3_reuse_session')\n@with_tempfile\n# forgot how to tell it not to change return value, so this side_effect beast now\n@patch.object(S3Authenticator, 'authenticate', side_effect=S3Authenticator.authenticate, autospec=True)\ndef test_reuse_session(tempfile, mocked_auth):\n    Providers.reset_default_providers()  # necessary for the testing below\n    providers = get_test_providers(url_2versions_nonversioned1_ver1)  # to check credentials\n    with swallow_outputs():\n        providers.download(url_2versions_nonversioned1_ver1, path=tempfile)\n    assert_equal(mocked_auth.call_count, 1)\n\n    providers2 = Providers.from_config_files()\n    with swallow_outputs():\n        providers2.download(url_2versions_nonversioned1_ver2, path=tempfile, overwrite=True)\n    assert_equal(mocked_auth.call_count, 1)\n\n    # but if we reload -- everything reloads and we need to authenticate again\n    providers2 = Providers.from_config_files(reload=True)\n    with swallow_outputs():\n        providers2.download(url_2versions_nonversioned1_ver2, path=tempfile, overwrite=True)\n    assert_equal(mocked_auth.call_count, 2)\n\n    Providers.reset_default_providers()  # necessary to avoid side-effects from having a vcr'ed connection\n    # leaking through default provider's bucket, e.g. breaking test_mtime if ran after this one\n\n\ndef test_parse_url():\n    from ..s3 import S3Downloader\n    f = S3Downloader._parse_url\n    b1 = \"s3://bucket.name/file/path?revision=123\"\n    assert_equal(f(b1, bucket_only=True), 'bucket.name')\n    assert_equal(f(b1), ('bucket.name', 'file/path', {'revision': '123'}))\n    assert_equal(f(\"s3://b/f name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%20name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%2Bname\"), ('b', 'f+name', {}))\n    assert_equal(f(\"s3://b/f%2bname?r=%20\"), ('b', 'f+name', {'r': '%20'}))\n\n\n@with_testsui(interactive=True)\ndef test_deny_access():\n    downloader = S3Downloader(authenticator=S3Authenticator())\n\n    def deny_access(*args, **kwargs):\n        raise AccessDeniedError\n\n    with assert_raises(DownloadError):\n        with patch.object(downloader, '_download', deny_access):\n            downloader.download(\"doesn't matter\")\n",
            "header": {
                "file": {
                    "path": "datalad/downloaders/tests/test_s3.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "3b1d7ec39",
                "new_commit_hash": "556f39df7",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/downloaders/tests/test_s3.py"
            },
            "plus_file": {
                "path": "b/datalad/downloaders/tests/test_s3.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 12,
                            "length": 6
                        },
                        "new_range": {
                            "start": 12,
                            "length": 7
                        },
                        "section": "import os"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "from mock import patch",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from ..s3 import S3Authenticator",
                                "type": "context"
                            },
                            {
                                "content": "from ..s3 import S3Downloader",
                                "type": "added"
                            },
                            {
                                "content": "from ..providers import Providers  # to test against crcns",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from ...tests.utils import swallow_outputs",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_s3.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ..s3 import S3Downloader",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 15,
                            "end_lineno": 15,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 22,
                            "length": 8
                        },
                        "new_range": {
                            "start": 23,
                            "length": 10
                        },
                        "section": "from ...tests.utils import assert_in"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "from ...tests.utils import use_cassette",
                                "type": "context"
                            },
                            {
                                "content": "from ...tests.utils import assert_raises",
                                "type": "context"
                            },
                            {
                                "content": "from ...tests.utils import skip_if_no_network",
                                "type": "context"
                            },
                            {
                                "content": "from ...tests.utils import with_testsui",
                                "type": "added"
                            },
                            {
                                "content": "from ...dochelpers import exc_str",
                                "type": "context"
                            },
                            {
                                "content": "from ...downloaders.base import DownloadError",
                                "type": "context"
                            },
                            {
                                "content": "from ...support.exceptions import AccessDeniedError",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "try:",
                                "type": "context"
                            },
                            {
                                "content": "    import boto",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_s3.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ...tests.utils import with_testsui",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 26,
                            "end_lineno": 26,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/downloaders/tests/test_s3.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ...support.exceptions import AccessDeniedError",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 29,
                            "end_lineno": 29,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 104,
                            "length": 4
                        },
                        "new_range": {
                            "start": 107,
                            "length": 16
                        },
                        "section": "def test_parse_url():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    assert_equal(f(\"s3://b/f name\"), ('b', 'f name', {}))",
                                "type": "context"
                            },
                            {
                                "content": "    assert_equal(f(\"s3://b/f%20name\"), ('b', 'f name', {}))",
                                "type": "context"
                            },
                            {
                                "content": "    assert_equal(f(\"s3://b/f%2Bname\"), ('b', 'f+name', {}))",
                                "type": "context"
                            },
                            {
                                "content": "    assert_equal(f(\"s3://b/f%2bname?r=%20\"), ('b', 'f+name', {'r': '%20'}))",
                                "type": "deleted"
                            },
                            {
                                "content": "No newline at end of file",
                                "type": "note"
                            },
                            {
                                "content": "    assert_equal(f(\"s3://b/f%2bname?r=%20\"), ('b', 'f+name', {'r': '%20'}))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "@with_testsui(interactive=True)",
                                "type": "added"
                            },
                            {
                                "content": "def test_deny_access():",
                                "type": "added"
                            },
                            {
                                "content": "    downloader = S3Downloader(authenticator=S3Authenticator())",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    def deny_access(*args, **kwargs):",
                                "type": "added"
                            },
                            {
                                "content": "        raise AccessDeniedError",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    with assert_raises(DownloadError):",
                                "type": "added"
                            },
                            {
                                "content": "        with patch.object(downloader, '_download', deny_access):",
                                "type": "added"
                            },
                            {
                                "content": "            downloader.download(\"doesn't matter\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_s3.py",
                            "type": "function",
                            "name": "test_parse_url",
                            "content": "def test_parse_url():\n    from ..s3 import S3Downloader\n    f = S3Downloader._parse_url\n    b1 = \"s3://bucket.name/file/path?revision=123\"\n    assert_equal(f(b1, bucket_only=True), 'bucket.name')\n    assert_equal(f(b1), ('bucket.name', 'file/path', {'revision': '123'}))\n    assert_equal(f(\"s3://b/f name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%20name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%2Bname\"), ('b', 'f+name', {}))\n    assert_equal(f(\"s3://b/f%2bname?r=%20\"), ('b', 'f+name', {'r': '%20'}))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 98,
                            "end_lineno": 107,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_s3.py",
                            "type": "function",
                            "name": "test_deny_access",
                            "content": "def test_deny_access():\n    downloader = S3Downloader(authenticator=S3Authenticator())\n\n    def deny_access(*args, **kwargs):\n        raise AccessDeniedError\n\n    with assert_raises(DownloadError):\n        with patch.object(downloader, '_download', deny_access):\n            downloader.download(\"doesn't matter\")",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 114,
                            "end_lineno": 122,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "07b952ca7ce075f49a6657225cd182455a21f818^",
    "new_commit_hash": "07b952ca7ce075f49a6657225cd182455a21f818",
    "commit_message": "Merge pull request #3091 from kyleam/enter-creds-typeerror\n\nBF: downloaders: Add guard against unset credential object",
    "commit_date": "2018-12-13T19:02:04-05:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "07b952ca7ce075f49a6657225cd182455a21f818",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for S3 downloader\"\"\"\n\nimport os\nfrom mock import patch\n\nfrom datalad.downloaders.s3 import S3Authenticator\nfrom datalad.downloaders.s3 import S3Downloader\nfrom datalad.downloaders.providers import Providers  # to test against crcns\n\nfrom datalad.tests.utils import swallow_outputs\nfrom datalad.tests.utils import SkipTest\nfrom datalad.tests.utils import with_tempfile\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_in\nfrom datalad.tests.utils import use_cassette\nfrom datalad.tests.utils import assert_raises\nfrom datalad.tests.utils import skip_if_no_network\nfrom datalad.tests.utils import with_testsui\nfrom datalad.dochelpers import exc_str\nfrom datalad.downloaders.base import DownloadError\nfrom datalad.support.exceptions import AccessDeniedError\n\ntry:\n    import boto\nexcept Exception as e:\n    raise SkipTest(\"boto module is not available: %s\" % exc_str(e))\n\nfrom datalad.downloaders.tests.utils import get_test_providers\nfrom datalad.downloaders.tests.test_http import check_download_external_url\n\nskip_if_no_network()  # TODO: provide persistent vcr fixtures for the tests\n\nurl_2versions_nonversioned1 = 's3://datalad-test0-versioned/2versions-nonversioned1.txt'\nurl_2versions_nonversioned1_ver1 = url_2versions_nonversioned1 + '?versionId=null'\nurl_2versions_nonversioned1_ver2 = url_2versions_nonversioned1 + '?versionId=V4Dqhu0QTEtxmvoNkCHGrjVZVomR1Ryo'\nurl_1version_bucketwithdot = 's3://datalad.test1/version1.txt'\n\n@use_cassette('test_s3_download_basic')\ndef test_s3_download_basic():\n\n    for url, success_str, failed_str in [\n        (url_2versions_nonversioned1, 'version2', 'version1'),\n        (url_2versions_nonversioned1_ver2, 'version2', 'version1'),\n        (url_2versions_nonversioned1_ver1, 'version1', 'version2'),\n        (url_1version_bucketwithdot, 'version1', 'nothing')\n    ]:\n        yield check_download_external_url, url, failed_str, success_str\n\n# TODO: redo smart way with mocking, to avoid unnecessary CPU waste\n@use_cassette('test_s3_mtime')\n@with_tempfile\ndef test_mtime(tempfile):\n    url = url_2versions_nonversioned1_ver2\n    with swallow_outputs():\n        # without allow_old=False it might be reusing previous connection\n        # which had already vcr tape for it, leading to failure.\n        # TODO:  make allow_old configurable and then within tests disallow\n        # allow_old\n        get_test_providers(url).download(url, path=tempfile, allow_old_session=False)\n    assert_equal(os.stat(tempfile).st_mtime, 1446873817)\n\n    # and if url is wrong\n    url = 's3://datalad-test0-versioned/nonexisting'\n    assert_raises(DownloadError, get_test_providers(url).download, url, path=tempfile, overwrite=True)\n\n\n@use_cassette('test_s3_reuse_session')\n@with_tempfile\n# forgot how to tell it not to change return value, so this side_effect beast now\n@patch.object(S3Authenticator, 'authenticate', side_effect=S3Authenticator.authenticate, autospec=True)\ndef test_reuse_session(tempfile, mocked_auth):\n    Providers.reset_default_providers()  # necessary for the testing below\n    providers = get_test_providers(url_2versions_nonversioned1_ver1)  # to check credentials\n    with swallow_outputs():\n        providers.download(url_2versions_nonversioned1_ver1, path=tempfile)\n    assert_equal(mocked_auth.call_count, 1)\n\n    providers2 = Providers.from_config_files()\n    with swallow_outputs():\n        providers2.download(url_2versions_nonversioned1_ver2, path=tempfile, overwrite=True)\n    assert_equal(mocked_auth.call_count, 1)\n\n    # but if we reload -- everything reloads and we need to authenticate again\n    providers2 = Providers.from_config_files(reload=True)\n    with swallow_outputs():\n        providers2.download(url_2versions_nonversioned1_ver2, path=tempfile, overwrite=True)\n    assert_equal(mocked_auth.call_count, 2)\n\n    Providers.reset_default_providers()  # necessary to avoid side-effects from having a vcr'ed connection\n    # leaking through default provider's bucket, e.g. breaking test_mtime if ran after this one\n\n\ndef test_parse_url():\n    from datalad.downloaders.s3 import S3Downloader\n    f = S3Downloader._parse_url\n    b1 = \"s3://bucket.name/file/path?revision=123\"\n    assert_equal(f(b1, bucket_only=True), 'bucket.name')\n    assert_equal(f(b1), ('bucket.name', 'file/path', {'revision': '123'}))\n    assert_equal(f(\"s3://b/f name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%20name\"), ('b', 'f name', {}))\n    assert_equal(f(\"s3://b/f%2Bname\"), ('b', 'f+name', {}))\n    assert_equal(f(\"s3://b/f%2bname?r=%20\"), ('b', 'f+name', {'r': '%20'}))\n\n\n@with_testsui(interactive=True)\ndef test_deny_access():\n    downloader = S3Downloader(authenticator=S3Authenticator())\n\n    def deny_access(*args, **kwargs):\n        raise AccessDeniedError\n\n    with assert_raises(DownloadError):\n        with patch.object(downloader, '_download', deny_access):\n            downloader.download(\"doesn't matter\")\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.11.1.dev88\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 54ms\nInstalled 25 packages in 5.67s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpiYfEm2/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpiYfEm2/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpiYfEm2/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpiYfEm2/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpiYfEm2/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 51, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWpb4D8/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWpb4D8/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWpb4D8/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWpb4D8/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWpb4D8/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 51, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/datalad/__init__.py\", line 43, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/datalad/cmd.py\", line 28, in <module>\n    from .support import path as op\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/datalad/support/path.py\", line 21, in <module>\n    from ..utils import (\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/datalad/utils.py\", line 26, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 303ms\nInstalled 26 packages in 3.60s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 57 packages in 1.53s\nPrepared 1 package in 1.16s\nInstalled 50 packages in 14ms\n + appdirs==1.4.4\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + cryptography==44.0.0\n + datalad==0.11.1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818)\n + deprecated==1.2.15\n + exifread==3.0.0\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + idna==3.10\n + importlib-resources==5.12.0\n + iso8601==2.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack==1.0.5\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.12\n + pillow==9.5.0\n + pycparser==2.21\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + requests==2.31.0\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.1\n + soupsieve==2.4.1\n + tqdm==4.67.1\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\nResolved 91 packages in 199ms\nInstalled 31 packages in 96ms\n + alabaster==0.7.13\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.14.0\n + build==1.1.1\n + distlib==0.3.9\n + docutils==0.19\n + filelock==3.12.2\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.9.16\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + platformdirs==4.0.0\n + psutil==6.1.1\n + pygments==2.17.2\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n + tabulate==0.9.0\n + virtualenv==20.26.6\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818\nplugins: cov-4.1.0\ncollected 5 items\n\nr2e_tests/test_1.py .EE..                                                [100%]\n\n==================================== ERRORS ====================================\n_________________________ ERROR at setup of test_mtime _________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py, line 58\n  @use_cassette('test_s3_mtime')\n  @with_tempfile\n  def test_mtime(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py:58\n_____________________ ERROR at setup of test_reuse_session _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py, line 75\n  @use_cassette('test_s3_reuse_session')\n  @with_tempfile\n  # forgot how to tell it not to change return value, so this side_effect beast now\n  @patch.object(S3Authenticator, 'authenticate', side_effect=S3Authenticator.authenticate, autospec=True)\n  def test_reuse_session(tempfile, mocked_auth):\nE       fixture 'mocked_auth' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py:75\n==================================== PASSES ====================================\n_______________________________ test_deny_access _______________________________\n----------------------------- Captured stderr call -----------------------------\n[INFO] Downloading \"doesn't matter\" into None \n[INFO] S3 session: Connecting to the bucket  anonymously \n------------------------------ Captured log call -------------------------------\nINFO     datalad.downloaders:base.py:467 Downloading \"doesn't matter\" into None\nINFO     datalad.http:s3.py:91 S3 session: Connecting to the bucket  anonymously\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_s3_download_basic\nPASSED r2e_tests/test_1.py::test_parse_url\nPASSED r2e_tests/test_1.py::test_deny_access\nERROR r2e_tests/test_1.py::test_mtime\nERROR r2e_tests/test_1.py::test_reuse_session\n========================= 3 passed, 2 errors in 1.16s ==========================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818\nplugins: cov-4.1.0\ncollected 5 items\n\nr2e_tests/test_1.py .EE.F                                                [100%]\n\n==================================== ERRORS ====================================\n_________________________ ERROR at setup of test_mtime _________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py, line 58\n  @use_cassette('test_s3_mtime')\n  @with_tempfile\n  def test_mtime(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py:58\n_____________________ ERROR at setup of test_reuse_session _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py, line 75\n  @use_cassette('test_s3_reuse_session')\n  @with_tempfile\n  # forgot how to tell it not to change return value, so this side_effect beast now\n  @patch.object(S3Authenticator, 'authenticate', side_effect=S3Authenticator.authenticate, autospec=True)\n  def test_reuse_session(tempfile, mocked_auth):\nE       fixture 'mocked_auth' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_07b952ca7ce075f49a6657225cd182455a21f818/r2e_tests/test_1.py:75\n=================================== FAILURES ===================================\n_______________________________ test_deny_access _______________________________\n\n    @with_testsui(interactive=True)\n    def test_deny_access():\n        downloader = S3Downloader(authenticator=S3Authenticator())\n    \n        def deny_access(*args, **kwargs):\n            raise AccessDeniedError\n    \n        with assert_raises(DownloadError):\n            with patch.object(downloader, '_download', deny_access):\n>               downloader.download(\"doesn't matter\")\n\nr2e_tests/test_1.py:122: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/downloaders/base.py:468: in download\n    return self.access(self._download, url, path=path, **kwargs)\ndatalad/downloaders/base.py:205: in access\n    new_provider=False)\ndatalad/downloaders/base.py:289: in _enter_credentials\n    text=\"Do you want to %s\" % action_msg):\ndatalad/ui/base.py:42: in yesno\n    response = self.question(*args, choices=['yes', 'no'], **kwargs).rstrip('\\n')\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def question(self, *args, **kwargs):\n        if not self._responses:\n            raise AssertionError(\n>               \"We are asked for a response whenever none is left to give\"\n            )\nE           AssertionError: We are asked for a response whenever none is left to give\n\ndatalad/ui/dialog.py:345: AssertionError\n----------------------------- Captured stderr call -----------------------------\n[INFO] Downloading \"doesn't matter\" into None \n[INFO] S3 session: Connecting to the bucket  anonymously \n------------------------------ Captured log call -------------------------------\nINFO     datalad.downloaders:base.py:467 Downloading \"doesn't matter\" into None\nINFO     datalad.http:s3.py:91 S3 session: Connecting to the bucket  anonymously\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_s3_download_basic\nPASSED r2e_tests/test_1.py::test_parse_url\nERROR r2e_tests/test_1.py::test_mtime\nERROR r2e_tests/test_1.py::test_reuse_session\nFAILED r2e_tests/test_1.py::test_deny_access - AssertionError: We are asked f...\n==================== 1 failed, 2 passed, 2 errors in 0.56s =====================\n",
    "old_commit_res_stderr": ""
}
datalad/downloaders/base.py datalad/downloaders/tests/test_s3.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Downloader prompts for credentials even when no credential object is set, causing AssertionError

**Description:**
When attempting to download a resource without initializing a credential object, the downloader unexpectedly prompts the user to enter credentials. Since no credential object is set, this leads to an `AssertionError` because the UI is asked for a response when none is available.

**Example Code:**
```python
from mock import patch
from datalad.downloaders.s3 import S3Downloader, S3Authenticator
from datalad.support.exceptions import AccessDeniedError
from datalad.downloaders.base import DownloadError
from datalad.tests.utils import assert_raises, with_testsui

@with_testsui(interactive=True)
def test_deny_access():
    downloader = S3Downloader(authenticator=S3Authenticator())

    def deny_access(*args, **kwargs):
        raise AccessDeniedError

    with assert_raises(DownloadError):
        with patch.object(downloader, '_download', deny_access):
            downloader.download("doesn't matter")
```

**Expected Behavior:**
When access is denied and no credential object is set, the downloader should raise a `DownloadError` without attempting to prompt the user for credentials.

**Actual Behavior:**
An `AssertionError` is raised with the message:
```
AssertionError: We are asked for a response whenever none is left to give
```
This occurs because the downloader tries to prompt for credentials even when the credential object is not set, leading to a crash.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/downloaders/base.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Provide access to stuff (html, data files) via HTTP and HTTPS

"""

__docformat__ = 'restructuredtext'

import msgpack
import os
import sys
import time

from abc import ABCMeta, abstractmethod
from os.path import exists, join as opj, isdir
from six import PY2
from six import binary_type, PY3
from six import add_metaclass
from six import reraise


from .. import cfg
from ..ui import ui
from ..utils import (
    auto_repr,
    unlink,
)
from ..dochelpers import exc_str
from .credentials import CREDENTIAL_TYPES
from ..support.exceptions import *
from ..support.network import RI

from logging import getLogger
lgr = getLogger('datalad.downloaders')


# TODO: remove headers, HTTP specific
@auto_repr
class DownloaderSession(object):
    """Base class to encapsulate information and possibly a session to download the content

    The idea is that corresponding downloader provides all necessary
    information and if necessary some kind of session to facilitate
    .download method
    """

    def __init__(self, size=None, filename=None, url=None, headers=None):
        self.size = size
        self.filename = filename
        self.headers = headers
        self.url = url

    def download(self, f=None, pbar=None, size=None):
        raise NotImplementedError("must be implemented in subclases")

        # TODO: get_status ?


@auto_repr
@add_metaclass(ABCMeta)
class BaseDownloader(object):
    """Base class for the downloaders"""

    _DEFAULT_AUTHENTICATOR = None
    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000

    def __init__(self, credential=None, authenticator=None):
        """

        Parameters
        ----------
        credential: Credential, optional
          Provides necessary credential fields to be used by authenticator
        authenticator: Authenticator, optional
          Authenticator to use for authentication.
        """
        if not authenticator and self._DEFAULT_AUTHENTICATOR:
            authenticator = self._DEFAULT_AUTHENTICATOR()

        if authenticator and authenticator.requires_authentication:
            if not credential and not authenticator.allows_anonymous:
                msg = "Both authenticator and credentials must be provided." \
                      " Got only authenticator %s" % repr(authenticator)
                if ui.yesno(
                    title=msg,
                    text="Do you want to enter %s credentials to be used?" % authenticator.DEFAULT_CREDENTIAL_TYPE
                ):
                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](
                        "session-only-for-%s" % id(authenticator))
                    credential.enter_new()
                    # TODO: give an option to store those credentials, and generate a basic provider
                    # record?
                else:
                    raise ValueError(msg)
        self.credential = credential
        self.authenticator = authenticator
        self._cache = None  # for fetches, not downloads

    def access(self, method, url, allow_old_session=True, **kwargs):
        """Generic decorator to manage access to the URL via some method

        Parameters
        ----------
        method : callable
          A callable, usually a method of the same class, which we decorate
          with access handling, and pass url as the first argument
        url : string
          URL to access
        *args, **kwargs
          Passed into the method call

        Returns
        -------
        None or bytes
        """
        # TODO: possibly wrap this logic outside within a decorator, which
        # would just call the corresponding method
        authenticator = self.authenticator
        if authenticator:
            needs_authentication = authenticator.requires_authentication
        else:
            needs_authentication = self.credential

        attempt, incomplete_attempt = 0, 0
        while True:
            attempt += 1
            if attempt > 20:
                # are we stuck in a loop somehow? I think logic doesn't allow this atm
                raise RuntimeError("Got to the %d'th iteration while trying to download %s" % (attempt, url))
            exc_info = None
            supported_auth_types = []
            try:
                used_old_session = False
                access_denied = False
                used_old_session = self._establish_session(url, allow_old=allow_old_session)
                if not allow_old_session:
                    assert(not used_old_session)
                lgr.log(5, "Calling out into %s for %s" % (method, url))
                result = method(url, **kwargs)
                # assume success if no puke etc
                break
            except AccessDeniedError as e:
                if isinstance(e, AnonymousAccessDeniedError):
                    access_denied = "Anonymous"
                else:
                    access_denied = "Authenticated"
                lgr.debug("%s access was denied: %s", access_denied, exc_str(e))
                supported_auth_types = e.supported_types
                exc_info = sys.exc_info()

            except IncompleteDownloadError as e:
                exc_info = sys.exc_info()
                incomplete_attempt += 1
                if incomplete_attempt > 5:
                    # give up
                    raise
                lgr.debug("Failed to download fully, will try again: %s", exc_str(e))
                # TODO: may be fail ealier than after 20 attempts in such a case?
            except DownloadError:
                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!
                raise

            msg_types = ''
            if supported_auth_types:
                msg_types = " The failure response indicated that following " \
                            "authentication types should be used: %s" % (', '.join(supported_auth_types))
            if access_denied:  # moved logic outside of except for clarity
                # TODO: what if it was anonimous attempt without authentication,
                #     so it is not "requires_authentication" but rather
                #     "supports_authentication"?  We should not report below in
                # _get_new_credential that authentication has failed then since there
                # were no authentication.  We might need a custom exception to
                # be caught above about that
                if needs_authentication:
                    # so we knew it needs authentication
                    if used_old_session:
                        # Let's try with fresh ones
                        allow_old_session = False
                        continue
                    else:
                        # we did use new cookies, we knew that authentication is needed
                        # but still failed. So possible cases:
                        #  1. authentication credentials changed/were revoked
                        #     - allow user to re-enter credentials
                        #  2. authentication mechanisms changed
                        #     - we can't do anything here about that
                        #  3. bug in out code which would render authentication/cookie handling
                        #     ineffective
                        #     - not sure what to do about it
                        if not ui.is_interactive:
                            lgr.error(
                                "Interface is non interactive, so we are "
                                "reraising: %s" % exc_str(e))
                            reraise(*exc_info)
                        self._enter_credentials(
                            url,
                            denied_msg=access_denied,
                            auth_types=supported_auth_types,
                            new_provider=False)
                        allow_old_session = False
                        continue
                else:  # None or False
                    if needs_authentication is False:
                        # those urls must or should NOT require authentication
                        # but we got denied
                        raise DownloadError(
                            "Failed to download from %s, which must be available"
                            "without authentication but access was denied. "
                            "Adjust your configuration for the provider.%s"
                            % (url, msg_types))
                    else:
                        # how could be None or any other non-False bool(False)
                        assert(needs_authentication is None)
                        # So we didn't know if authentication necessary, and it
                        # seems to be necessary, so Let's ask the user to setup
                        # authentication mechanism for this website
                        self._enter_credentials(
                            url,
                            denied_msg=access_denied,
                            auth_types=supported_auth_types,
                            new_provider=True)
                        allow_old_session = False
                        continue

        return result

    def _setup_new_provider(self, title, url, auth_types=None):
        # Full new provider (TODO move into Providers?)
        from .providers import Providers
        providers = Providers.from_config_files()
        while True:
            provider = providers.enter_new(url, auth_types=auth_types)
            if not provider:
                if ui.yesno(
                    title="Re-enter provider?",
                    text="You haven't entered or saved provider, would you like to retry?",
                    default=True
                ):
                    continue
            break
        return provider


    def _enter_credentials(
            self, url, denied_msg,
            auth_types=[], new_provider=True):
        """Use when authentication fails to set new credentials for url

        Raises
        ------
        DownloadError
          If no known credentials type or user refuses to update
        """
        title = "{msg} access to {url} has failed.".format(
            msg=denied_msg, url=url)

        if new_provider:
            # No credential was known, we need to create an
            # appropriate one
            if not ui.yesno(
                    title=title,
                    text="Would you like to setup a new provider configuration"
                         " to access url?",
                    default=True
            ):
                assert not self.authenticator, "bug: incorrect assumption"
                raise DownloadError(
                    title +
                    " No authenticator is known, cannot set any credential")
            else:
                provider = self._setup_new_provider(
                    title, url, auth_types=auth_types)
                self.authenticator = provider.authenticator
                self.credential = provider.credential
                if not (self.credential and self.credential.is_known):
                    # TODO: or should we ask to re-enter?
                    self.credential.enter_new()
        else:
            action_msg = "enter other credentials in case they were updated?"

            if ui.yesno(
                    title=title,
                    text="Do you want to %s" % action_msg):
                self.credential.enter_new()
            else:
                raise DownloadError(
                    "Failed to download from %s given available credentials"
                    % url)

    @staticmethod
    def _get_temp_download_filename(filepath):
        """Given a filepath, return the one to use as temp file during download
        """
        # TODO: might better reside somewhere under .datalad/tmp or
        # .git/datalad/tmp
        return filepath + ".datalad-download-temp"

    @abstractmethod
    def get_downloader_session(self, url):
        """

        Parameters
        ----------
        url : str

        Returns
        -------
        downloader_into_fp: callable
           Which takes two parameters: file, pbar
        target_size: int or None (if unknown)
        url: str
           Possibly redirected url
        url_filename: str or None
           Filename as decided from the (redirected) url
        headers : dict or None
        """
        raise NotImplementedError("Must be implemented in the subclass")

    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):
        """Verify that download finished correctly"""

        if (self.authenticator
                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \
                and hasattr(self.authenticator, 'failure_re') \
                and self.authenticator.failure_re:
            assert hasattr(self.authenticator, 'check_for_auth_failure'), \
                "%s has failure_re defined but no check_for_auth_failure" \
                % self.authenticator

            if file_:
                with open(file_) as fp:
                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)
            else:
                assert(content is not None)

            self.authenticator.check_for_auth_failure(
                content, "Download of the url %s has failed: " % url)

        if target_size and target_size != downloaded_size:
            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(
                "Downloaded size %d differs from originally announced %d" % (downloaded_size, target_size))

    def _download(self, url, path=None, overwrite=False, size=None, stats=None):
        """Download content into a file

        Parameters
        ----------
        url: str
          URL to download
        path: str, optional
          Path to file where to store the downloaded content.  If None,
          filename deduced from the url and saved in curdir
        size: int, optional
          Limit in size to be downloaded

        Returns
        -------
        None or string
          Returns downloaded filename

        """

        downloader_session = self.get_downloader_session(url)
        status = self.get_status_from_headers(downloader_session.headers)

        target_size = downloader_session.size
        if size is not None:
            target_size = min(target_size, size)

        #### Specific to download
        if path:
            if isdir(path):
                # provided path is a directory under which to save
                filename = downloader_session.filename
                if not filename:
                    raise DownloadError(
                        "File name could not be determined from {}".format(url))
                filepath = opj(path, filename)
            else:
                filepath = path
        else:
            filepath = downloader_session.filename

        existed = exists(filepath)
        if existed and not overwrite:
            raise DownloadError("File %s already exists" % filepath)

        # FETCH CONTENT
        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])
        try:
            temp_filepath = self._get_temp_download_filename(filepath)
            if exists(temp_filepath):
                # eventually we might want to continue the download
                lgr.warning(
                    "Temporary file %s from the previous download was found. "
                    "It will be overriden" % temp_filepath)
                # TODO.  also logic below would clean it up atm

            with open(temp_filepath, 'wb') as fp:
                # TODO: url might be a bit too long for the beast.
                # Consider to improve to make it animated as well, or shorten here
                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)
                t0 = time.time()
                downloader_session.download(fp, pbar, size=size)
                downloaded_time = time.time() - t0
                pbar.finish()
            downloaded_size = os.stat(temp_filepath).st_size

            # (headers.get('Content-type', "") and headers.get('Content-Type')).startswith('text/html')
            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator
            self._verify_download(url, downloaded_size, target_size, temp_filepath)

            # adjust atime/mtime according to headers/status
            if status.mtime:
                lgr.log(5, "Setting mtime for %s to be %s", temp_filepath, status.mtime)
                os.utime(temp_filepath, (time.time(), status.mtime))

            # place successfully downloaded over the filepath
            os.rename(temp_filepath, filepath)

            if stats:
                stats.downloaded += 1
                stats.overwritten += int(existed)
                stats.downloaded_size += downloaded_size
                stats.downloaded_time += downloaded_time
        except (AccessDeniedError, IncompleteDownloadError) as e:
            raise
        except Exception as e:
            e_str = exc_str(e, limit=5)
            lgr.error("Failed to download {url} into {filepath}: {e_str}".format(
                **locals()
            ))
            raise DownloadError(exc_str(e))  # for now
        finally:
            if exists(temp_filepath):
                # clean up
                lgr.debug("Removing a temporary download %s", temp_filepath)
                unlink(temp_filepath)

        return filepath

    def download(self, url, path=None, **kwargs):
        """Fetch content as pointed by the URL optionally into a file

        Parameters
        ----------
        url : string
          URL to access
        path : string, optional
          Filename or existing directory to store downloaded content under.
          If not provided -- deduced from the url

        Returns
        -------
        string
          file path
        """
        # TODO: may be move all the path dealing logic here
        # but then it might require sending request anyways for Content-Disposition
        # so probably nah
        lgr.info("Downloading %r into %r", url, path)
        return self.access(self._download, url, path=path, **kwargs)

    @property
    def cache(self):
        if self._cache is None:
            # TODO: move this all logic outside into a dedicated caching beast
            lgr.info("Initializing cache for fetches")
            if PY2:
                import anydbm as dbm
            else:
                import dbm
            # Initiate cache.
            # Very rudimentary caching for now, might fail many ways
            cache_dir = cfg.obtain('datalad.locations.cache')
            if not exists(cache_dir):
                os.makedirs(cache_dir)
            cache_path = opj(cache_dir, 'crawl_cache.dbm')
            self._cache = dbm.open(cache_path, 'c')
            import atexit
            atexit.register(self._cache.close)
        return self._cache

    def _fetch(self, url, cache=None, size=None, allow_redirects=True):
        """Fetch content from a url into a file.

        Very similar to _download but lacks any "file" management and decodes
        content

        Parameters
        ----------
        url: str
          URL to download
        cache: bool, optional
          If None, config is consulted to determine whether results should be
          cached. Cache is operating based on url, so no verification of any
          kind is carried out

        Returns
        -------
        bytes, dict
          content, headers
        """
        lgr.log(3, "_fetch(%r, cache=%r, size=%r, allow_redirects=%r)",
                url, cache, size, allow_redirects)
        if cache is None:
            cache = cfg.obtain('datalad.crawl.cache', default=False)

        if cache:
            cache_key = msgpack.dumps(url)
            lgr.debug("Loading content for url %s from cache", url)
            res = self.cache.get(cache_key)
            if res is not None:
                try:
                    return msgpack.loads(res, encoding='utf-8')
                except Exception as exc:
                    lgr.warning("Failed to unpack loaded from cache for %s: %s",
                                url, exc_str(exc))

        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)

        target_size = downloader_session.size
        if size is not None:
            if size == 0:
                # no download of the content was requested -- just return headers and be done
                return None, downloader_session.headers
            target_size = min(size, target_size)

        # FETCH CONTENT
        try:
            # Consider to improve to make it animated as well, or shorten here
            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)
            content = downloader_session.download(size=size)
            #pbar.finish()
            downloaded_size = len(content)

            # now that we know size based on encoded content, let's decode into string type
            if PY3 and isinstance(content, binary_type):
                content = content.decode()
            # downloaded_size = os.stat(temp_filepath).st_size

            self._verify_download(url, downloaded_size, target_size, None, content=content)

        except (AccessDeniedError, IncompleteDownloadError) as e:
            raise
        except Exception as e:
            e_str = exc_str(e, limit=5)
            lgr.error("Failed to fetch {url}: {e_str}".format(**locals()))
            raise DownloadError(exc_str(e, limit=8))  # for now

        if cache:
            # apparently requests' CaseInsensitiveDict is not serialazable
            # TODO:  may be we should reuse that type everywhere, to avoid
            # out own handling for case-handling
            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))

        return content, downloader_session.headers

    def fetch(self, url, **kwargs):
        """Fetch and return content (not decoded) as pointed by the URL

        Parameters
        ----------
        url : string
          URL to access

        Returns
        -------
        bytes
          content
        """
        lgr.info("Fetching %r", url)
        # Do not return headers, just content
        out = self.access(self._fetch, url, **kwargs)
        return out[0]

    def get_status(self, url, old_status=None, **kwargs):
        """Return status of the url as a dict, None if N/A

        Parameters
        ----------
        url : string
          URL to access
        old_status : FileStatus, optional
          Previous status record.  If provided, might serve as a shortcut
          to assess if status has changed, and if not -- return the same
          record

        Returns
        -------
        dict
          dict-like beast depicting the status of the URL if accessible.
          Returned value should be sufficient to tell if the URL content
          has changed by comparing to previously obtained value.
          If URL is not reachable, None would be returned
        """
        return self.access(self._get_status, url, old_status=old_status, **kwargs)

    # TODO: borrow from itself... ?
    # @borrowkwargs(BaseDownloader, 'get_status')
    def _get_status(self, url, old_status=None):

        # the tricky part is only to make sure that we are getting the target URL
        # and not some page saying to login, that is why we need to fetch some content
        # in those cases, and not just check the headers
        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \
            if self.authenticator \
            and hasattr(self.authenticator, 'failure_re') \
            and self.authenticator.failure_re \
            else 0

        _, headers = self._fetch(url, cache=False, size=download_size)

        # extract from headers information to depict the status of the url
        status = self.get_status_from_headers(headers)

        if old_status is not None:
            raise NotImplementedError("Do not know yet how to deal with old_status. TODO")

        return status

    @classmethod
    @abstractmethod
    def get_status_from_headers(cls, headers):
        raise NotImplementedError("Implement in the subclass: %s" % cls)

    def get_target_url(self, url):
        """Return url after possible redirections

        Parameters
        ----------
        url : string
          URL to access

        Returns
        -------
        str
        """
        return self.access(self._get_target_url, url)

    def _get_target_url(self, url):
        return self.get_downloader_session(url).url


#
# Authenticators    XXX might go into authenticators.py
#

class Authenticator(object):
    """Abstract common class for different types of authentication

    Derived classes should get parameterized with options from the config files
    from "provider:" sections
    """
    requires_authentication = True
    allows_anonymous = False
    # TODO: figure out interface

    DEFAULT_CREDENTIAL_TYPE = 'user_password'

    def authenticate(self, *args, **kwargs):
        """Derived classes will provide specific implementation
        """
        if self.requires_authentication:
            raise NotImplementedError("Authentication for %s not yet implemented" % self.__class__)


class NotImplementedAuthenticator(Authenticator):
    pass


class NoneAuthenticator(Authenticator):
    """Whenever no authentication is necessary and that is stated explicitly"""
    requires_authentication = False
    pass

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Downloader prompts for credentials even when no credential object is set, causing AssertionError

**Description:**
When attempting to download a resource without initializing a credential object, the downloader unexpectedly prompts the user to enter credentials. Since no credential object is set, this leads to an `AssertionError` because the UI is asked for a response when none is available.

**Example Code:**
```python
from mock import patch
from datalad.downloaders.s3 import S3Downloader, S3Authenticator
from datalad.support.exceptions import AccessDeniedError
from datalad.downloaders.base import DownloadError
from datalad.tests.utils import assert_raises, with_testsui

@with_testsui(interactive=True)
def test_deny_access():
    downloader = S3Downloader(authenticator=S3Authenticator())

    def deny_access(*args, **kwargs):
        raise AccessDeniedError

    with assert_raises(DownloadError):
        with patch.object(downloader, '_download', deny_access):
            downloader.download("doesn't matter")
```

**Expected Behavior:**
When access is denied and no credential object is set, the downloader should raise a `DownloadError` without attempting to prompt the user for credentials.

**Actual Behavior:**
An `AssertionError` is raised with the message:
```
AssertionError: We are asked for a response whenever none is left to give
```
This occurs because the downloader tries to prompt for credentials even when the credential object is not set, leading to a crash.

[/ISSUE]
{
    "test_parse_url": "PASSED",
    "test_deny_access": "PASSED",
    "test_mtime": "ERROR",
    "test_reuse_session": "ERROR"
}
diff --git a/datalad/downloaders/base.py b/datalad/downloaders/base.py
index 37e09fa25..eafe16a70 100644
--- a/datalad/downloaders/base.py
+++ b/datalad/downloaders/base.py
@@ -284,7 +284,7 @@ class BaseDownloader(object):
         else:
             action_msg = "enter other credentials in case they were updated?"
 
-            if ui.yesno(
+            if self.credential and ui.yesno(
                     title=title,
                     text="Do you want to %s" % action_msg):
                 self.credential.enter_new()

{"datalad/downloaders/base.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Provide access to stuff (html, data files) via HTTP and HTTPS\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport msgpack\nimport os\nimport sys\nimport time\n\nfrom abc import ABCMeta, abstractmethod\nfrom os.path import exists, join as opj, isdir\nfrom six import PY2\nfrom six import binary_type, PY3\nfrom six import add_metaclass\nfrom six import reraise\n\n\nfrom .. import cfg\nfrom ..ui import ui\nfrom ..utils import (\n    auto_repr,\n    unlink,\n)\nfrom ..dochelpers import exc_str\nfrom .credentials import CREDENTIAL_TYPES\nfrom ..support.exceptions import *\nfrom ..support.network import RI\n\nfrom logging import getLogger\nlgr = getLogger('datalad.downloaders')\n\n\n# TODO: remove headers, HTTP specific\n@auto_repr\nclass DownloaderSession(object):\n    \"\"\"Base class to encapsulate information and possibly a session to download the content\n\n    The idea is that corresponding downloader provides all necessary\n    information and if necessary some kind of session to facilitate\n    .download method\n    \"\"\"\n\n    def __init__(self, size=None, filename=None, url=None, headers=None):\n        self.size = size\n        self.filename = filename\n        self.headers = headers\n        self.url = url\n\n    def download(self, f=None, pbar=None, size=None):\n        raise NotImplementedError(\"must be implemented in subclases\")\n\n        # TODO: get_status ?\n\n\n@auto_repr\n@add_metaclass(ABCMeta)\nclass BaseDownloader(object):\n    \"\"\"Base class for the downloaders\"\"\"\n\n    _DEFAULT_AUTHENTICATOR = None\n    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000\n\n    def __init__(self, credential=None, authenticator=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        credential: Credential, optional\n          Provides necessary credential fields to be used by authenticator\n        authenticator: Authenticator, optional\n          Authenticator to use for authentication.\n        \"\"\"\n        if not authenticator and self._DEFAULT_AUTHENTICATOR:\n            authenticator = self._DEFAULT_AUTHENTICATOR()\n\n        if authenticator and authenticator.requires_authentication:\n            if not credential and not authenticator.allows_anonymous:\n                msg = \"Both authenticator and credentials must be provided.\" \\\n                      \" Got only authenticator %s\" % repr(authenticator)\n                if ui.yesno(\n                    title=msg,\n                    text=\"Do you want to enter %s credentials to be used?\" % authenticator.DEFAULT_CREDENTIAL_TYPE\n                ):\n                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](\n                        \"session-only-for-%s\" % id(authenticator))\n                    credential.enter_new()\n                    # TODO: give an option to store those credentials, and generate a basic provider\n                    # record?\n                else:\n                    raise ValueError(msg)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._cache = None  # for fetches, not downloads\n\n    def access(self, method, url, allow_old_session=True, **kwargs):\n        \"\"\"Generic decorator to manage access to the URL via some method\n\n        Parameters\n        ----------\n        method : callable\n          A callable, usually a method of the same class, which we decorate\n          with access handling, and pass url as the first argument\n        url : string\n          URL to access\n        *args, **kwargs\n          Passed into the method call\n\n        Returns\n        -------\n        None or bytes\n        \"\"\"\n        # TODO: possibly wrap this logic outside within a decorator, which\n        # would just call the corresponding method\n        authenticator = self.authenticator\n        if authenticator:\n            needs_authentication = authenticator.requires_authentication\n        else:\n            needs_authentication = self.credential\n\n        attempt, incomplete_attempt = 0, 0\n        while True:\n            attempt += 1\n            if attempt > 20:\n                # are we stuck in a loop somehow? I think logic doesn't allow this atm\n                raise RuntimeError(\"Got to the %d'th iteration while trying to download %s\" % (attempt, url))\n            exc_info = None\n            supported_auth_types = []\n            try:\n                used_old_session = False\n                access_denied = False\n                used_old_session = self._establish_session(url, allow_old=allow_old_session)\n                if not allow_old_session:\n                    assert(not used_old_session)\n                lgr.log(5, \"Calling out into %s for %s\" % (method, url))\n                result = method(url, **kwargs)\n                # assume success if no puke etc\n                break\n            except AccessDeniedError as e:\n                if isinstance(e, AnonymousAccessDeniedError):\n                    access_denied = \"Anonymous\"\n                else:\n                    access_denied = \"Authenticated\"\n                lgr.debug(\"%s access was denied: %s\", access_denied, exc_str(e))\n                supported_auth_types = e.supported_types\n                exc_info = sys.exc_info()\n\n            except IncompleteDownloadError as e:\n                exc_info = sys.exc_info()\n                incomplete_attempt += 1\n                if incomplete_attempt > 5:\n                    # give up\n                    raise\n                lgr.debug(\"Failed to download fully, will try again: %s\", exc_str(e))\n                # TODO: may be fail ealier than after 20 attempts in such a case?\n            except DownloadError:\n                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!\n                raise\n\n            msg_types = ''\n            if supported_auth_types:\n                msg_types = \" The failure response indicated that following \" \\\n                            \"authentication types should be used: %s\" % (', '.join(supported_auth_types))\n            if access_denied:  # moved logic outside of except for clarity\n                # TODO: what if it was anonimous attempt without authentication,\n                #     so it is not \"requires_authentication\" but rather\n                #     \"supports_authentication\"?  We should not report below in\n                # _get_new_credential that authentication has failed then since there\n                # were no authentication.  We might need a custom exception to\n                # be caught above about that\n                if needs_authentication:\n                    # so we knew it needs authentication\n                    if used_old_session:\n                        # Let's try with fresh ones\n                        allow_old_session = False\n                        continue\n                    else:\n                        # we did use new cookies, we knew that authentication is needed\n                        # but still failed. So possible cases:\n                        #  1. authentication credentials changed/were revoked\n                        #     - allow user to re-enter credentials\n                        #  2. authentication mechanisms changed\n                        #     - we can't do anything here about that\n                        #  3. bug in out code which would render authentication/cookie handling\n                        #     ineffective\n                        #     - not sure what to do about it\n                        if not ui.is_interactive:\n                            lgr.error(\n                                \"Interface is non interactive, so we are \"\n                                \"reraising: %s\" % exc_str(e))\n                            reraise(*exc_info)\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=False)\n                        allow_old_session = False\n                        continue\n                else:  # None or False\n                    if needs_authentication is False:\n                        # those urls must or should NOT require authentication\n                        # but we got denied\n                        raise DownloadError(\n                            \"Failed to download from %s, which must be available\"\n                            \"without authentication but access was denied. \"\n                            \"Adjust your configuration for the provider.%s\"\n                            % (url, msg_types))\n                    else:\n                        # how could be None or any other non-False bool(False)\n                        assert(needs_authentication is None)\n                        # So we didn't know if authentication necessary, and it\n                        # seems to be necessary, so Let's ask the user to setup\n                        # authentication mechanism for this website\n                        self._enter_credentials(\n                            url,\n                            denied_msg=access_denied,\n                            auth_types=supported_auth_types,\n                            new_provider=True)\n                        allow_old_session = False\n                        continue\n\n        return result\n\n    def _setup_new_provider(self, title, url, auth_types=None):\n        # Full new provider (TODO move into Providers?)\n        from .providers import Providers\n        providers = Providers.from_config_files()\n        while True:\n            provider = providers.enter_new(url, auth_types=auth_types)\n            if not provider:\n                if ui.yesno(\n                    title=\"Re-enter provider?\",\n                    text=\"You haven't entered or saved provider, would you like to retry?\",\n                    default=True\n                ):\n                    continue\n            break\n        return provider\n\n\n    def _enter_credentials(\n            self, url, denied_msg,\n            auth_types=[], new_provider=True):\n        \"\"\"Use when authentication fails to set new credentials for url\n\n        Raises\n        ------\n        DownloadError\n          If no known credentials type or user refuses to update\n        \"\"\"\n        title = \"{msg} access to {url} has failed.\".format(\n            msg=denied_msg, url=url)\n\n        if new_provider:\n            # No credential was known, we need to create an\n            # appropriate one\n            if not ui.yesno(\n                    title=title,\n                    text=\"Would you like to setup a new provider configuration\"\n                         \" to access url?\",\n                    default=True\n            ):\n                assert not self.authenticator, \"bug: incorrect assumption\"\n                raise DownloadError(\n                    title +\n                    \" No authenticator is known, cannot set any credential\")\n            else:\n                provider = self._setup_new_provider(\n                    title, url, auth_types=auth_types)\n                self.authenticator = provider.authenticator\n                self.credential = provider.credential\n                if not (self.credential and self.credential.is_known):\n                    # TODO: or should we ask to re-enter?\n                    self.credential.enter_new()\n        else:\n            action_msg = \"enter other credentials in case they were updated?\"\n\n            if ui.yesno(\n                    title=title,\n                    text=\"Do you want to %s\" % action_msg):\n                self.credential.enter_new()\n            else:\n                raise DownloadError(\n                    \"Failed to download from %s given available credentials\"\n                    % url)\n\n    @staticmethod\n    def _get_temp_download_filename(filepath):\n        \"\"\"Given a filepath, return the one to use as temp file during download\n        \"\"\"\n        # TODO: might better reside somewhere under .datalad/tmp or\n        # .git/datalad/tmp\n        return filepath + \".datalad-download-temp\"\n\n    @abstractmethod\n    def get_downloader_session(self, url):\n        \"\"\"\n\n        Parameters\n        ----------\n        url : str\n\n        Returns\n        -------\n        downloader_into_fp: callable\n           Which takes two parameters: file, pbar\n        target_size: int or None (if unknown)\n        url: str\n           Possibly redirected url\n        url_filename: str or None\n           Filename as decided from the (redirected) url\n        headers : dict or None\n        \"\"\"\n        raise NotImplementedError(\"Must be implemented in the subclass\")\n\n    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):\n        \"\"\"Verify that download finished correctly\"\"\"\n\n        if (self.authenticator\n                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \\\n                and hasattr(self.authenticator, 'failure_re') \\\n                and self.authenticator.failure_re:\n            assert hasattr(self.authenticator, 'check_for_auth_failure'), \\\n                \"%s has failure_re defined but no check_for_auth_failure\" \\\n                % self.authenticator\n\n            if file_:\n                with open(file_) as fp:\n                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)\n            else:\n                assert(content is not None)\n\n            self.authenticator.check_for_auth_failure(\n                content, \"Download of the url %s has failed: \" % url)\n\n        if target_size and target_size != downloaded_size:\n            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(\n                \"Downloaded size %d differs from originally announced %d\" % (downloaded_size, target_size))\n\n    def _download(self, url, path=None, overwrite=False, size=None, stats=None):\n        \"\"\"Download content into a file\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        path: str, optional\n          Path to file where to store the downloaded content.  If None,\n          filename deduced from the url and saved in curdir\n        size: int, optional\n          Limit in size to be downloaded\n\n        Returns\n        -------\n        None or string\n          Returns downloaded filename\n\n        \"\"\"\n\n        downloader_session = self.get_downloader_session(url)\n        status = self.get_status_from_headers(downloader_session.headers)\n\n        target_size = downloader_session.size\n        if size is not None:\n            target_size = min(target_size, size)\n\n        #### Specific to download\n        if path:\n            if isdir(path):\n                # provided path is a directory under which to save\n                filename = downloader_session.filename\n                if not filename:\n                    raise DownloadError(\n                        \"File name could not be determined from {}\".format(url))\n                filepath = opj(path, filename)\n            else:\n                filepath = path\n        else:\n            filepath = downloader_session.filename\n\n        existed = exists(filepath)\n        if existed and not overwrite:\n            raise DownloadError(\"File %s already exists\" % filepath)\n\n        # FETCH CONTENT\n        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])\n        try:\n            temp_filepath = self._get_temp_download_filename(filepath)\n            if exists(temp_filepath):\n                # eventually we might want to continue the download\n                lgr.warning(\n                    \"Temporary file %s from the previous download was found. \"\n                    \"It will be overriden\" % temp_filepath)\n                # TODO.  also logic below would clean it up atm\n\n            with open(temp_filepath, 'wb') as fp:\n                # TODO: url might be a bit too long for the beast.\n                # Consider to improve to make it animated as well, or shorten here\n                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n                t0 = time.time()\n                downloader_session.download(fp, pbar, size=size)\n                downloaded_time = time.time() - t0\n                pbar.finish()\n            downloaded_size = os.stat(temp_filepath).st_size\n\n            # (headers.get('Content-type', \"\") and headers.get('Content-Type')).startswith('text/html')\n            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator\n            self._verify_download(url, downloaded_size, target_size, temp_filepath)\n\n            # adjust atime/mtime according to headers/status\n            if status.mtime:\n                lgr.log(5, \"Setting mtime for %s to be %s\", temp_filepath, status.mtime)\n                os.utime(temp_filepath, (time.time(), status.mtime))\n\n            # place successfully downloaded over the filepath\n            os.rename(temp_filepath, filepath)\n\n            if stats:\n                stats.downloaded += 1\n                stats.overwritten += int(existed)\n                stats.downloaded_size += downloaded_size\n                stats.downloaded_time += downloaded_time\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to download {url} into {filepath}: {e_str}\".format(\n                **locals()\n            ))\n            raise DownloadError(exc_str(e))  # for now\n        finally:\n            if exists(temp_filepath):\n                # clean up\n                lgr.debug(\"Removing a temporary download %s\", temp_filepath)\n                unlink(temp_filepath)\n\n        return filepath\n\n    def download(self, url, path=None, **kwargs):\n        \"\"\"Fetch content as pointed by the URL optionally into a file\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        path : string, optional\n          Filename or existing directory to store downloaded content under.\n          If not provided -- deduced from the url\n\n        Returns\n        -------\n        string\n          file path\n        \"\"\"\n        # TODO: may be move all the path dealing logic here\n        # but then it might require sending request anyways for Content-Disposition\n        # so probably nah\n        lgr.info(\"Downloading %r into %r\", url, path)\n        return self.access(self._download, url, path=path, **kwargs)\n\n    @property\n    def cache(self):\n        if self._cache is None:\n            # TODO: move this all logic outside into a dedicated caching beast\n            lgr.info(\"Initializing cache for fetches\")\n            if PY2:\n                import anydbm as dbm\n            else:\n                import dbm\n            # Initiate cache.\n            # Very rudimentary caching for now, might fail many ways\n            cache_dir = cfg.obtain('datalad.locations.cache')\n            if not exists(cache_dir):\n                os.makedirs(cache_dir)\n            cache_path = opj(cache_dir, 'crawl_cache.dbm')\n            self._cache = dbm.open(cache_path, 'c')\n            import atexit\n            atexit.register(self._cache.close)\n        return self._cache\n\n    def _fetch(self, url, cache=None, size=None, allow_redirects=True):\n        \"\"\"Fetch content from a url into a file.\n\n        Very similar to _download but lacks any \"file\" management and decodes\n        content\n\n        Parameters\n        ----------\n        url: str\n          URL to download\n        cache: bool, optional\n          If None, config is consulted to determine whether results should be\n          cached. Cache is operating based on url, so no verification of any\n          kind is carried out\n\n        Returns\n        -------\n        bytes, dict\n          content, headers\n        \"\"\"\n        lgr.log(3, \"_fetch(%r, cache=%r, size=%r, allow_redirects=%r)\",\n                url, cache, size, allow_redirects)\n        if cache is None:\n            cache = cfg.obtain('datalad.crawl.cache', default=False)\n\n        if cache:\n            cache_key = msgpack.dumps(url)\n            lgr.debug(\"Loading content for url %s from cache\", url)\n            res = self.cache.get(cache_key)\n            if res is not None:\n                try:\n                    return msgpack.loads(res, encoding='utf-8')\n                except Exception as exc:\n                    lgr.warning(\"Failed to unpack loaded from cache for %s: %s\",\n                                url, exc_str(exc))\n\n        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)\n\n        target_size = downloader_session.size\n        if size is not None:\n            if size == 0:\n                # no download of the content was requested -- just return headers and be done\n                return None, downloader_session.headers\n            target_size = min(size, target_size)\n\n        # FETCH CONTENT\n        try:\n            # Consider to improve to make it animated as well, or shorten here\n            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)\n            content = downloader_session.download(size=size)\n            #pbar.finish()\n            downloaded_size = len(content)\n\n            # now that we know size based on encoded content, let's decode into string type\n            if PY3 and isinstance(content, binary_type):\n                content = content.decode()\n            # downloaded_size = os.stat(temp_filepath).st_size\n\n            self._verify_download(url, downloaded_size, target_size, None, content=content)\n\n        except (AccessDeniedError, IncompleteDownloadError) as e:\n            raise\n        except Exception as e:\n            e_str = exc_str(e, limit=5)\n            lgr.error(\"Failed to fetch {url}: {e_str}\".format(**locals()))\n            raise DownloadError(exc_str(e, limit=8))  # for now\n\n        if cache:\n            # apparently requests' CaseInsensitiveDict is not serialazable\n            # TODO:  may be we should reuse that type everywhere, to avoid\n            # out own handling for case-handling\n            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))\n\n        return content, downloader_session.headers\n\n    def fetch(self, url, **kwargs):\n        \"\"\"Fetch and return content (not decoded) as pointed by the URL\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        bytes\n          content\n        \"\"\"\n        lgr.info(\"Fetching %r\", url)\n        # Do not return headers, just content\n        out = self.access(self._fetch, url, **kwargs)\n        return out[0]\n\n    def get_status(self, url, old_status=None, **kwargs):\n        \"\"\"Return status of the url as a dict, None if N/A\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n        old_status : FileStatus, optional\n          Previous status record.  If provided, might serve as a shortcut\n          to assess if status has changed, and if not -- return the same\n          record\n\n        Returns\n        -------\n        dict\n          dict-like beast depicting the status of the URL if accessible.\n          Returned value should be sufficient to tell if the URL content\n          has changed by comparing to previously obtained value.\n          If URL is not reachable, None would be returned\n        \"\"\"\n        return self.access(self._get_status, url, old_status=old_status, **kwargs)\n\n    # TODO: borrow from itself... ?\n    # @borrowkwargs(BaseDownloader, 'get_status')\n    def _get_status(self, url, old_status=None):\n\n        # the tricky part is only to make sure that we are getting the target URL\n        # and not some page saying to login, that is why we need to fetch some content\n        # in those cases, and not just check the headers\n        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \\\n            if self.authenticator \\\n            and hasattr(self.authenticator, 'failure_re') \\\n            and self.authenticator.failure_re \\\n            else 0\n\n        _, headers = self._fetch(url, cache=False, size=download_size)\n\n        # extract from headers information to depict the status of the url\n        status = self.get_status_from_headers(headers)\n\n        if old_status is not None:\n            raise NotImplementedError(\"Do not know yet how to deal with old_status. TODO\")\n\n        return status\n\n    @classmethod\n    @abstractmethod\n    def get_status_from_headers(cls, headers):\n        raise NotImplementedError(\"Implement in the subclass: %s\" % cls)\n\n    def get_target_url(self, url):\n        \"\"\"Return url after possible redirections\n\n        Parameters\n        ----------\n        url : string\n          URL to access\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return self.access(self._get_target_url, url)\n\n    def _get_target_url(self, url):\n        return self.get_downloader_session(url).url\n\n\n#\n# Authenticators    XXX might go into authenticators.py\n#\n\nclass Authenticator(object):\n    \"\"\"Abstract common class for different types of authentication\n\n    Derived classes should get parameterized with options from the config files\n    from \"provider:\" sections\n    \"\"\"\n    requires_authentication = True\n    allows_anonymous = False\n    # TODO: figure out interface\n\n    DEFAULT_CREDENTIAL_TYPE = 'user_password'\n\n    def authenticate(self, *args, **kwargs):\n        \"\"\"Derived classes will provide specific implementation\n        \"\"\"\n        if self.requires_authentication:\n            raise NotImplementedError(\"Authentication for %s not yet implemented\" % self.__class__)\n\n\nclass NotImplementedAuthenticator(Authenticator):\n    pass\n\n\nclass NoneAuthenticator(Authenticator):\n    \"\"\"Whenever no authentication is necessary and that is stated explicitly\"\"\"\n    requires_authentication = False\n    pass\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Downloader prompts for credentials even when no credential object is set, causing AssertionError

**Description:**
When attempting to download a resource without initializing a credential object, the downloader unexpectedly prompts the user to enter credentials. Since no credential object is set, this leads to an `AssertionError` because the UI is asked for a response when none is available.

**Example Code:**
```python
from mock import patch
from datalad.downloaders.s3 import S3Downloader, S3Authenticator
from datalad.support.exceptions import AccessDeniedError
from datalad.downloaders.base import DownloadError
from datalad.tests.utils import assert_raises, with_testsui

@with_testsui(interactive=True)
def test_deny_access():
    downloader = S3Downloader(authenticator=S3Authenticator())

    def deny_access(*args, **kwargs):
        raise AccessDeniedError

    with assert_raises(DownloadError):
        with patch.object(downloader, '_download', deny_access):
            downloader.download("doesn't matter")
```

**Expected Behavior:**
When access is denied and no credential object is set, the downloader should raise a `DownloadError` without attempting to prompt the user for credentials.

**Actual Behavior:**
An `AssertionError` is raised with the message:
```
AssertionError: We are asked for a response whenever none is left to give
```
This occurs because the downloader tries to prompt for credentials even when the credential object is not set, leading to a crash.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/downloaders/base.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Provide access to stuff (html, data files) via HTTP and HTTPS

"""

__docformat__ = 'restructuredtext'

import msgpack
import os
import sys
import time

from abc import ABCMeta, abstractmethod
from os.path import exists, join as opj, isdir
from six import PY2
from six import binary_type, PY3
from six import add_metaclass
from six import reraise


from .. import cfg
from ..ui import ui
from ..utils import (
    auto_repr,
    unlink,
)
from ..dochelpers import exc_str
from .credentials import CREDENTIAL_TYPES
from ..support.exceptions import *
from ..support.network import RI

from logging import getLogger
lgr = getLogger('datalad.downloaders')


# TODO: remove headers, HTTP specific
@auto_repr
class DownloaderSession(object):
    """Base class to encapsulate information and possibly a session to download the content

    The idea is that corresponding downloader provides all necessary
    information and if necessary some kind of session to facilitate
    .download method
    """

    def __init__(self, size=None, filename=None, url=None, headers=None):
        self.size = size
        self.filename = filename
        self.headers = headers
        self.url = url

    def download(self, f=None, pbar=None, size=None):
        raise NotImplementedError("must be implemented in subclases")

        # TODO: get_status ?


@auto_repr
@add_metaclass(ABCMeta)
class BaseDownloader(object):
    """Base class for the downloaders"""

    _DEFAULT_AUTHENTICATOR = None
    _DOWNLOAD_SIZE_TO_VERIFY_AUTH = 10000

    def __init__(self, credential=None, authenticator=None):
        """

        Parameters
        ----------
        credential: Credential, optional
          Provides necessary credential fields to be used by authenticator
        authenticator: Authenticator, optional
          Authenticator to use for authentication.
        """
        if not authenticator and self._DEFAULT_AUTHENTICATOR:
            authenticator = self._DEFAULT_AUTHENTICATOR()

        if authenticator and authenticator.requires_authentication:
            if not credential and not authenticator.allows_anonymous:
                msg = "Both authenticator and credentials must be provided." \
                      " Got only authenticator %s" % repr(authenticator)
                if ui.yesno(
                    title=msg,
                    text="Do you want to enter %s credentials to be used?" % authenticator.DEFAULT_CREDENTIAL_TYPE
                ):
                    credential = CREDENTIAL_TYPES[authenticator.DEFAULT_CREDENTIAL_TYPE](
                        "session-only-for-%s" % id(authenticator))
                    credential.enter_new()
                    # TODO: give an option to store those credentials, and generate a basic provider
                    # record?
                else:
                    raise ValueError(msg)
        self.credential = credential
        self.authenticator = authenticator
        self._cache = None  # for fetches, not downloads

    def access(self, method, url, allow_old_session=True, **kwargs):
        """Generic decorator to manage access to the URL via some method

        Parameters
        ----------
        method : callable
          A callable, usually a method of the same class, which we decorate
          with access handling, and pass url as the first argument
        url : string
          URL to access
        *args, **kwargs
          Passed into the method call

        Returns
        -------
        None or bytes
        """
        # TODO: possibly wrap this logic outside within a decorator, which
        # would just call the corresponding method
        authenticator = self.authenticator
        if authenticator:
            needs_authentication = authenticator.requires_authentication
        else:
            needs_authentication = self.credential

        attempt, incomplete_attempt = 0, 0
        while True:
            attempt += 1
            if attempt > 20:
                # are we stuck in a loop somehow? I think logic doesn't allow this atm
                raise RuntimeError("Got to the %d'th iteration while trying to download %s" % (attempt, url))
            exc_info = None
            supported_auth_types = []
            try:
                used_old_session = False
                access_denied = False
                used_old_session = self._establish_session(url, allow_old=allow_old_session)
                if not allow_old_session:
                    assert(not used_old_session)
                lgr.log(5, "Calling out into %s for %s" % (method, url))
                result = method(url, **kwargs)
                # assume success if no puke etc
                break
            except AccessDeniedError as e:
                if isinstance(e, AnonymousAccessDeniedError):
                    access_denied = "Anonymous"
                else:
                    access_denied = "Authenticated"
                lgr.debug("%s access was denied: %s", access_denied, exc_str(e))
                supported_auth_types = e.supported_types
                exc_info = sys.exc_info()

            except IncompleteDownloadError as e:
                exc_info = sys.exc_info()
                incomplete_attempt += 1
                if incomplete_attempt > 5:
                    # give up
                    raise
                lgr.debug("Failed to download fully, will try again: %s", exc_str(e))
                # TODO: may be fail ealier than after 20 attempts in such a case?
            except DownloadError:
                # TODO Handle some known ones, possibly allow for a few retries, otherwise just let it go!
                raise

            msg_types = ''
            if supported_auth_types:
                msg_types = " The failure response indicated that following " \
                            "authentication types should be used: %s" % (', '.join(supported_auth_types))
            if access_denied:  # moved logic outside of except for clarity
                # TODO: what if it was anonimous attempt without authentication,
                #     so it is not "requires_authentication" but rather
                #     "supports_authentication"?  We should not report below in
                # _get_new_credential that authentication has failed then since there
                # were no authentication.  We might need a custom exception to
                # be caught above about that
                if needs_authentication:
                    # so we knew it needs authentication
                    if used_old_session:
                        # Let's try with fresh ones
                        allow_old_session = False
                        continue
                    else:
                        # we did use new cookies, we knew that authentication is needed
                        # but still failed. So possible cases:
                        #  1. authentication credentials changed/were revoked
                        #     - allow user to re-enter credentials
                        #  2. authentication mechanisms changed
                        #     - we can't do anything here about that
                        #  3. bug in out code which would render authentication/cookie handling
                        #     ineffective
                        #     - not sure what to do about it
                        if not ui.is_interactive:
                            lgr.error(
                                "Interface is non interactive, so we are "
                                "reraising: %s" % exc_str(e))
                            reraise(*exc_info)
                        self._enter_credentials(
                            url,
                            denied_msg=access_denied,
                            auth_types=supported_auth_types,
                            new_provider=False)
                        allow_old_session = False
                        continue
                else:  # None or False
                    if needs_authentication is False:
                        # those urls must or should NOT require authentication
                        # but we got denied
                        raise DownloadError(
                            "Failed to download from %s, which must be available"
                            "without authentication but access was denied. "
                            "Adjust your configuration for the provider.%s"
                            % (url, msg_types))
                    else:
                        # how could be None or any other non-False bool(False)
                        assert(needs_authentication is None)
                        # So we didn't know if authentication necessary, and it
                        # seems to be necessary, so Let's ask the user to setup
                        # authentication mechanism for this website
                        self._enter_credentials(
                            url,
                            denied_msg=access_denied,
                            auth_types=supported_auth_types,
                            new_provider=True)
                        allow_old_session = False
                        continue

        return result

    def _setup_new_provider(self, title, url, auth_types=None):
        # Full new provider (TODO move into Providers?)
        from .providers import Providers
        providers = Providers.from_config_files()
        while True:
            provider = providers.enter_new(url, auth_types=auth_types)
            if not provider:
                if ui.yesno(
                    title="Re-enter provider?",
                    text="You haven't entered or saved provider, would you like to retry?",
                    default=True
                ):
                    continue
            break
        return provider


    def _enter_credentials(
            self, url, denied_msg,
            auth_types=[], new_provider=True):
        """Use when authentication fails to set new credentials for url

        Raises
        ------
        DownloadError
          If no known credentials type or user refuses to update
        """
        title = "{msg} access to {url} has failed.".format(
            msg=denied_msg, url=url)

        if new_provider:
            # No credential was known, we need to create an
            # appropriate one
            if not ui.yesno(
                    title=title,
                    text="Would you like to setup a new provider configuration"
                         " to access url?",
                    default=True
            ):
                assert not self.authenticator, "bug: incorrect assumption"
                raise DownloadError(
                    title +
                    " No authenticator is known, cannot set any credential")
            else:
                provider = self._setup_new_provider(
                    title, url, auth_types=auth_types)
                self.authenticator = provider.authenticator
                self.credential = provider.credential
                if not (self.credential and self.credential.is_known):
                    # TODO: or should we ask to re-enter?
                    self.credential.enter_new()
        else:
            action_msg = "enter other credentials in case they were updated?"

            if ui.yesno(
                    title=title,
                    text="Do you want to %s" % action_msg):
                self.credential.enter_new()
            else:
                raise DownloadError(
                    "Failed to download from %s given available credentials"
                    % url)

    @staticmethod
    def _get_temp_download_filename(filepath):
        """Given a filepath, return the one to use as temp file during download
        """
        # TODO: might better reside somewhere under .datalad/tmp or
        # .git/datalad/tmp
        return filepath + ".datalad-download-temp"

    @abstractmethod
    def get_downloader_session(self, url):
        """

        Parameters
        ----------
        url : str

        Returns
        -------
        downloader_into_fp: callable
           Which takes two parameters: file, pbar
        target_size: int or None (if unknown)
        url: str
           Possibly redirected url
        url_filename: str or None
           Filename as decided from the (redirected) url
        headers : dict or None
        """
        raise NotImplementedError("Must be implemented in the subclass")

    def _verify_download(self, url, downloaded_size, target_size, file_=None, content=None):
        """Verify that download finished correctly"""

        if (self.authenticator
                and downloaded_size < self._DOWNLOAD_SIZE_TO_VERIFY_AUTH) \
                and hasattr(self.authenticator, 'failure_re') \
                and self.authenticator.failure_re:
            assert hasattr(self.authenticator, 'check_for_auth_failure'), \
                "%s has failure_re defined but no check_for_auth_failure" \
                % self.authenticator

            if file_:
                with open(file_) as fp:
                    content = fp.read(self._DOWNLOAD_SIZE_TO_VERIFY_AUTH)
            else:
                assert(content is not None)

            self.authenticator.check_for_auth_failure(
                content, "Download of the url %s has failed: " % url)

        if target_size and target_size != downloaded_size:
            raise (IncompleteDownloadError if target_size > downloaded_size else UnaccountedDownloadError)(
                "Downloaded size %d differs from originally announced %d" % (downloaded_size, target_size))

    def _download(self, url, path=None, overwrite=False, size=None, stats=None):
        """Download content into a file

        Parameters
        ----------
        url: str
          URL to download
        path: str, optional
          Path to file where to store the downloaded content.  If None,
          filename deduced from the url and saved in curdir
        size: int, optional
          Limit in size to be downloaded

        Returns
        -------
        None or string
          Returns downloaded filename

        """

        downloader_session = self.get_downloader_session(url)
        status = self.get_status_from_headers(downloader_session.headers)

        target_size = downloader_session.size
        if size is not None:
            target_size = min(target_size, size)

        #### Specific to download
        if path:
            if isdir(path):
                # provided path is a directory under which to save
                filename = downloader_session.filename
                if not filename:
                    raise DownloadError(
                        "File name could not be determined from {}".format(url))
                filepath = opj(path, filename)
            else:
                filepath = path
        else:
            filepath = downloader_session.filename

        existed = exists(filepath)
        if existed and not overwrite:
            raise DownloadError("File %s already exists" % filepath)

        # FETCH CONTENT
        # TODO: pbar = ui.get_progressbar(size=response.headers['size'])
        try:
            temp_filepath = self._get_temp_download_filename(filepath)
            if exists(temp_filepath):
                # eventually we might want to continue the download
                lgr.warning(
                    "Temporary file %s from the previous download was found. "
                    "It will be overriden" % temp_filepath)
                # TODO.  also logic below would clean it up atm

            with open(temp_filepath, 'wb') as fp:
                # TODO: url might be a bit too long for the beast.
                # Consider to improve to make it animated as well, or shorten here
                pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)
                t0 = time.time()
                downloader_session.download(fp, pbar, size=size)
                downloaded_time = time.time() - t0
                pbar.finish()
            downloaded_size = os.stat(temp_filepath).st_size

            # (headers.get('Content-type', "") and headers.get('Content-Type')).startswith('text/html')
            #  and self.authenticator.html_form_failure_re: # TODO: use information in authenticator
            self._verify_download(url, downloaded_size, target_size, temp_filepath)

            # adjust atime/mtime according to headers/status
            if status.mtime:
                lgr.log(5, "Setting mtime for %s to be %s", temp_filepath, status.mtime)
                os.utime(temp_filepath, (time.time(), status.mtime))

            # place successfully downloaded over the filepath
            os.rename(temp_filepath, filepath)

            if stats:
                stats.downloaded += 1
                stats.overwritten += int(existed)
                stats.downloaded_size += downloaded_size
                stats.downloaded_time += downloaded_time
        except (AccessDeniedError, IncompleteDownloadError) as e:
            raise
        except Exception as e:
            e_str = exc_str(e, limit=5)
            lgr.error("Failed to download {url} into {filepath}: {e_str}".format(
                **locals()
            ))
            raise DownloadError(exc_str(e))  # for now
        finally:
            if exists(temp_filepath):
                # clean up
                lgr.debug("Removing a temporary download %s", temp_filepath)
                unlink(temp_filepath)

        return filepath

    def download(self, url, path=None, **kwargs):
        """Fetch content as pointed by the URL optionally into a file

        Parameters
        ----------
        url : string
          URL to access
        path : string, optional
          Filename or existing directory to store downloaded content under.
          If not provided -- deduced from the url

        Returns
        -------
        string
          file path
        """
        # TODO: may be move all the path dealing logic here
        # but then it might require sending request anyways for Content-Disposition
        # so probably nah
        lgr.info("Downloading %r into %r", url, path)
        return self.access(self._download, url, path=path, **kwargs)

    @property
    def cache(self):
        if self._cache is None:
            # TODO: move this all logic outside into a dedicated caching beast
            lgr.info("Initializing cache for fetches")
            if PY2:
                import anydbm as dbm
            else:
                import dbm
            # Initiate cache.
            # Very rudimentary caching for now, might fail many ways
            cache_dir = cfg.obtain('datalad.locations.cache')
            if not exists(cache_dir):
                os.makedirs(cache_dir)
            cache_path = opj(cache_dir, 'crawl_cache.dbm')
            self._cache = dbm.open(cache_path, 'c')
            import atexit
            atexit.register(self._cache.close)
        return self._cache

    def _fetch(self, url, cache=None, size=None, allow_redirects=True):
        """Fetch content from a url into a file.

        Very similar to _download but lacks any "file" management and decodes
        content

        Parameters
        ----------
        url: str
          URL to download
        cache: bool, optional
          If None, config is consulted to determine whether results should be
          cached. Cache is operating based on url, so no verification of any
          kind is carried out

        Returns
        -------
        bytes, dict
          content, headers
        """
        lgr.log(3, "_fetch(%r, cache=%r, size=%r, allow_redirects=%r)",
                url, cache, size, allow_redirects)
        if cache is None:
            cache = cfg.obtain('datalad.crawl.cache', default=False)

        if cache:
            cache_key = msgpack.dumps(url)
            lgr.debug("Loading content for url %s from cache", url)
            res = self.cache.get(cache_key)
            if res is not None:
                try:
                    return msgpack.loads(res, encoding='utf-8')
                except Exception as exc:
                    lgr.warning("Failed to unpack loaded from cache for %s: %s",
                                url, exc_str(exc))

        downloader_session = self.get_downloader_session(url, allow_redirects=allow_redirects)

        target_size = downloader_session.size
        if size is not None:
            if size == 0:
                # no download of the content was requested -- just return headers and be done
                return None, downloader_session.headers
            target_size = min(size, target_size)

        # FETCH CONTENT
        try:
            # Consider to improve to make it animated as well, or shorten here
            #pbar = ui.get_progressbar(label=url, fill_text=filepath, total=target_size)
            content = downloader_session.download(size=size)
            #pbar.finish()
            downloaded_size = len(content)

            # now that we know size based on encoded content, let's decode into string type
            if PY3 and isinstance(content, binary_type):
                content = content.decode()
            # downloaded_size = os.stat(temp_filepath).st_size

            self._verify_download(url, downloaded_size, target_size, None, content=content)

        except (AccessDeniedError, IncompleteDownloadError) as e:
            raise
        except Exception as e:
            e_str = exc_str(e, limit=5)
            lgr.error("Failed to fetch {url}: {e_str}".format(**locals()))
            raise DownloadError(exc_str(e, limit=8))  # for now

        if cache:
            # apparently requests' CaseInsensitiveDict is not serialazable
            # TODO:  may be we should reuse that type everywhere, to avoid
            # out own handling for case-handling
            self.cache[cache_key] = msgpack.dumps((content, dict(downloader_session.headers)))

        return content, downloader_session.headers

    def fetch(self, url, **kwargs):
        """Fetch and return content (not decoded) as pointed by the URL

        Parameters
        ----------
        url : string
          URL to access

        Returns
        -------
        bytes
          content
        """
        lgr.info("Fetching %r", url)
        # Do not return headers, just content
        out = self.access(self._fetch, url, **kwargs)
        return out[0]

    def get_status(self, url, old_status=None, **kwargs):
        """Return status of the url as a dict, None if N/A

        Parameters
        ----------
        url : string
          URL to access
        old_status : FileStatus, optional
          Previous status record.  If provided, might serve as a shortcut
          to assess if status has changed, and if not -- return the same
          record

        Returns
        -------
        dict
          dict-like beast depicting the status of the URL if accessible.
          Returned value should be sufficient to tell if the URL content
          has changed by comparing to previously obtained value.
          If URL is not reachable, None would be returned
        """
        return self.access(self._get_status, url, old_status=old_status, **kwargs)

    # TODO: borrow from itself... ?
    # @borrowkwargs(BaseDownloader, 'get_status')
    def _get_status(self, url, old_status=None):

        # the tricky part is only to make sure that we are getting the target URL
        # and not some page saying to login, that is why we need to fetch some content
        # in those cases, and not just check the headers
        download_size = self._DOWNLOAD_SIZE_TO_VERIFY_AUTH \
            if self.authenticator \
            and hasattr(self.authenticator, 'failure_re') \
            and self.authenticator.failure_re \
            else 0

        _, headers = self._fetch(url, cache=False, size=download_size)

        # extract from headers information to depict the status of the url
        status = self.get_status_from_headers(headers)

        if old_status is not None:
            raise NotImplementedError("Do not know yet how to deal with old_status. TODO")

        return status

    @classmethod
    @abstractmethod
    def get_status_from_headers(cls, headers):
        raise NotImplementedError("Implement in the subclass: %s" % cls)

    def get_target_url(self, url):
        """Return url after possible redirections

        Parameters
        ----------
        url : string
          URL to access

        Returns
        -------
        str
        """
        return self.access(self._get_target_url, url)

    def _get_target_url(self, url):
        return self.get_downloader_session(url).url


#
# Authenticators    XXX might go into authenticators.py
#

class Authenticator(object):
    """Abstract common class for different types of authentication

    Derived classes should get parameterized with options from the config files
    from "provider:" sections
    """
    requires_authentication = True
    allows_anonymous = False
    # TODO: figure out interface

    DEFAULT_CREDENTIAL_TYPE = 'user_password'

    def authenticate(self, *args, **kwargs):
        """Derived classes will provide specific implementation
        """
        if self.requires_authentication:
            raise NotImplementedError("Authentication for %s not yet implemented" % self.__class__)


class NotImplementedAuthenticator(Authenticator):
    pass


class NoneAuthenticator(Authenticator):
    """Whenever no authentication is necessary and that is stated explicitly"""
    requires_authentication = False
    pass

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user