{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Data providers - bind downloaders and credentials together\n\"\"\"\n\nimport os\n\nfrom glob import glob\nfrom logging import getLogger\nfrom six import iteritems\n\nimport re\nfrom os.path import dirname, abspath, join as pathjoin\nfrom six.moves.urllib.parse import urlparse\n\nfrom .base import NoneAuthenticator, NotImplementedAuthenticator\n\nfrom .http import HTMLFormAuthenticator, HTTPBasicAuthAuthenticator, HTTPDigestAuthAuthenticator\nfrom .http import HTTPDownloader\nfrom .s3 import S3Authenticator, S3Downloader\nfrom ..support.configparserinc import SafeConfigParserWithIncludes\nfrom ..support.external_versions import external_versions\nfrom ..utils import assure_list_from_str\nfrom ..utils import auto_repr\nfrom ..utils import get_dataset_root\n\nfrom ..interface.common_cfg import dirs\n\nlgr = getLogger('datalad.downloaders.providers')\n\n# dict to bind authentication_type's to authenticator classes\n# parameters will be fetched from config file itself\nAUTHENTICATION_TYPES = {\n    'html_form': HTMLFormAuthenticator,\n    'http_auth': HTTPBasicAuthAuthenticator,\n    'http_basic_auth': HTTPBasicAuthAuthenticator,\n    'http_digest_auth': HTTPDigestAuthAuthenticator,\n    'aws-s3': S3Authenticator,  # TODO: check if having '-' is kosher\n    'nda-s3': S3Authenticator,\n    'xnat': NotImplementedAuthenticator,\n    'none': NoneAuthenticator,\n}\n\nfrom .credentials import CREDENTIAL_TYPES\n\ndef resolve_url_to_name(d, url):\n    \"\"\"Given a directory (e.g. of SiteInformation._items or Credential._items)\n    go through url_re and find the corresponding item and returns its key (i.e. name)\n    \"\"\"\n\n    for k, rec in iteritems(d):\n        for url_re in rec.get('url_re', '').split('\\n'):\n            if url_re:\n                if re.search(url_re, url):\n                    return k\n    return None\n\n\n@auto_repr\nclass Provider(object):\n    \"\"\"Class to bring together url_res, credential, and authenticator\n    \"\"\"\n    # TODO: we might need a lazy loading of the submodules which would provide\n    # specific downloaders while importing needed Python modules \"on demand\"\n    DOWNLOADERS = {\n        'http': {'class': HTTPDownloader, 'externals': {'requests'}},\n        'https': {'class': HTTPDownloader, 'externals': {'requests'}},\n        'ftp': {'class': HTTPDownloader, 'externals': {'requests', 'boto'}},\n        's3': {'class': S3Downloader, 'externals': {'boto'}}\n        # ... TODO\n    }\n\n    def __init__(self, name, url_res, credential=None, authenticator=None,\n                 downloader=None):\n        \"\"\"\n        Parameters\n        ----------\n        name: str\n        url_res: list of str\n           Regular expressions\n        credential: Credential, optional\n        authenticator: Authenticator, optional\n        downloader: Downloader, optional\n\n        \"\"\"\n        self.name = name\n        self.url_res = assure_list_from_str(url_res)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._downloader = downloader\n\n    @property\n    def downloader(self):\n        return self._downloader\n\n    @staticmethod\n    def get_scheme_from_url(url):\n        \"\"\"Given a URL return scheme to decide which downloader class to use\n        \"\"\"\n        url_split = urlparse(url)\n        return url_split.scheme  # , url_split.netloc)\n\n    @classmethod\n    def _get_downloader_class(cls, url):\n        key = cls.get_scheme_from_url(url)\n        if key in cls.DOWNLOADERS:\n            entry = cls.DOWNLOADERS[key]\n            klass = entry['class']\n            for ext in entry.get('externals', set()):\n                if external_versions[ext] is None:\n                    raise RuntimeError(\n                        \"For using %s downloader, you need '%s' dependency \"\n                        \"which seems to be missing\" % (klass, ext)\n                    )\n            return klass\n        else:\n            raise ValueError(\"Do not know how to handle url %s for scheme %s. Known: %s\"\n                             % (url, key, cls.DOWNLOADERS.keys()))\n\n    def get_downloader(self, url, **kwargs):\n        \"\"\"Assigns proper downloader given the URL\n\n        If one is known -- verifies its appropriateness for the given url.\n        ATM we do not support multiple types of downloaders per single provider\n        \"\"\"\n        if self._downloader is None:\n            # we need to create a new one\n            Downloader = self._get_downloader_class(url)\n            # we might need to provide it with credentials and authenticator\n            # Let's do via kwargs so we could accomodate cases when downloader does not necessarily\n            # cares about those... duck typing or what it is in action\n            kwargs = kwargs.copy()\n            if self.credential:\n                kwargs['credential'] = self.credential\n            if self.authenticator:\n                kwargs['authenticator'] = self.authenticator\n            self._downloader = Downloader(**kwargs)\n        return self._downloader\n\n\nclass Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None\n\n\n    # # TODO: UNUSED?\n    # def get_credentials(self, url, new=False):\n    #     \"\"\"Ask user to enter credentials for a provider matching url\n    #     \"\"\"\n    #     # find a match among _items\n    #     provider = self.get_provider(url)\n    #     if new or not provider:\n    #         rec = self._get_new_record_ui(url)\n    #         rec['url_re'] = \"TODO\"  # present to user and ask to edit\n    #         name = urlparse(url).netloc\n    #         self._items[name] = rec\n    #         if ui.yesno(\"Do you want to store credentials for %s\" % name):\n    #             self.store_credentials()\n    #     else:\n    #         return self._items[name]\n    #\n    # def store_credentials(self, name):\n    #     # TODO: store  self._items[name]  in appropriate (user) creds\n    #     # for later reuse\n    #     raise NotImplementedError()\n    #\n    # def _get_new_record_ui(self, url):\n    #     # TODO: should be a dialog with the fields appropriate for this particular\n    #     # type of credentials\n    #     ui.message(\"To access %s we would need credentials.\" % url)\n    #     if url in self.providers:\n    #         ui.message(\"If you don't yet have credentials, please visit %s\"\n    #                    % self.providers.get(url, 'credentials_url'))\n    #     return { 'user': ui.question(\"Username:\"),\n    #              'password': ui.password() }\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Data providers - bind downloaders and credentials together\n\"\"\"\n\nimport os\n\nfrom glob import glob\nfrom logging import getLogger\nfrom six import iteritems\n\nimport re\nfrom os.path import dirname, abspath, join as pathjoin\nfrom six.moves.urllib.parse import urlparse\nfrom collections import OrderedDict\n\nfrom .base import NoneAuthenticator, NotImplementedAuthenticator\n\nfrom .http import HTMLFormAuthenticator, HTTPBasicAuthAuthenticator, HTTPDigestAuthAuthenticator\nfrom .http import HTTPDownloader\nfrom .s3 import S3Authenticator, S3Downloader\nfrom ..support.configparserinc import SafeConfigParserWithIncludes\nfrom ..support.external_versions import external_versions\nfrom ..utils import assure_list_from_str\nfrom ..utils import auto_repr\nfrom ..utils import get_dataset_root\n\nfrom ..interface.common_cfg import dirs\n\nlgr = getLogger('datalad.downloaders.providers')\n\n# dict to bind authentication_type's to authenticator classes\n# parameters will be fetched from config file itself\nAUTHENTICATION_TYPES = {\n    'html_form': HTMLFormAuthenticator,\n    'http_auth': HTTPBasicAuthAuthenticator,\n    'http_basic_auth': HTTPBasicAuthAuthenticator,\n    'http_digest_auth': HTTPDigestAuthAuthenticator,\n    'aws-s3': S3Authenticator,  # TODO: check if having '-' is kosher\n    'nda-s3': S3Authenticator,\n    'xnat': NotImplementedAuthenticator,\n    'none': NoneAuthenticator,\n}\n\nfrom .credentials import CREDENTIAL_TYPES\n\ndef resolve_url_to_name(d, url):\n    \"\"\"Given a directory (e.g. of SiteInformation._items or Credential._items)\n    go through url_re and find the corresponding item and returns its key (i.e. name)\n    \"\"\"\n\n    for k, rec in iteritems(d):\n        for url_re in rec.get('url_re', '').split('\\n'):\n            if url_re:\n                if re.search(url_re, url):\n                    return k\n    return None\n\n\n@auto_repr\nclass Provider(object):\n    \"\"\"Class to bring together url_res, credential, and authenticator\n    \"\"\"\n    # TODO: we might need a lazy loading of the submodules which would provide\n    # specific downloaders while importing needed Python modules \"on demand\"\n    DOWNLOADERS = {\n        'http': {'class': HTTPDownloader, 'externals': {'requests'}},\n        'https': {'class': HTTPDownloader, 'externals': {'requests'}},\n        'ftp': {'class': HTTPDownloader, 'externals': {'requests', 'boto'}},\n        's3': {'class': S3Downloader, 'externals': {'boto'}}\n        # ... TODO\n    }\n\n    def __init__(self, name, url_res, credential=None, authenticator=None,\n                 downloader=None):\n        \"\"\"\n        Parameters\n        ----------\n        name: str\n        url_res: list of str\n           Regular expressions\n        credential: Credential, optional\n        authenticator: Authenticator, optional\n        downloader: Downloader, optional\n\n        \"\"\"\n        self.name = name\n        self.url_res = assure_list_from_str(url_res)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._downloader = downloader\n\n    @property\n    def downloader(self):\n        return self._downloader\n\n    @staticmethod\n    def get_scheme_from_url(url):\n        \"\"\"Given a URL return scheme to decide which downloader class to use\n        \"\"\"\n        url_split = urlparse(url)\n        return url_split.scheme  # , url_split.netloc)\n\n    @classmethod\n    def _get_downloader_class(cls, url):\n        key = cls.get_scheme_from_url(url)\n        if key in cls.DOWNLOADERS:\n            entry = cls.DOWNLOADERS[key]\n            klass = entry['class']\n            for ext in entry.get('externals', set()):\n                if external_versions[ext] is None:\n                    raise RuntimeError(\n                        \"For using %s downloader, you need '%s' dependency \"\n                        \"which seems to be missing\" % (klass, ext)\n                    )\n            return klass\n        else:\n            raise ValueError(\"Do not know how to handle url %s for scheme %s. Known: %s\"\n                             % (url, key, cls.DOWNLOADERS.keys()))\n\n    def get_downloader(self, url, **kwargs):\n        \"\"\"Assigns proper downloader given the URL\n\n        If one is known -- verifies its appropriateness for the given url.\n        ATM we do not support multiple types of downloaders per single provider\n        \"\"\"\n        if self._downloader is None:\n            # we need to create a new one\n            Downloader = self._get_downloader_class(url)\n            # we might need to provide it with credentials and authenticator\n            # Let's do via kwargs so we could accomodate cases when downloader does not necessarily\n            # cares about those... duck typing or what it is in action\n            kwargs = kwargs.copy()\n            if self.credential:\n                kwargs['credential'] = self.credential\n            if self.authenticator:\n                kwargs['authenticator'] = self.authenticator\n            self._downloader = Downloader(**kwargs)\n        return self._downloader\n\n\nclass Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n\n            # System config\n            if dirs.site_config_dir is not None:\n                files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n            # User config\n            if dirs.user_config_dir is not None:\n                files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        # Order matters, because we need to ensure that when\n        # there's a conflict between configuration files declared\n        # at different precedence levels (ie. dataset vs system)\n        # the appropriate precedence config wins.\n        providers = OrderedDict()\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n\n        # Range backwards to ensure that more locally defined\n        # configuration wins in conflicts between url_re\n        for i in range(nproviders-1, -1, -1):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    #if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        #del self._providers[i]\n                        #self._providers = [provider] + self._providers\n                        #assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None\n\n\n    # # TODO: UNUSED?\n    # def get_credentials(self, url, new=False):\n    #     \"\"\"Ask user to enter credentials for a provider matching url\n    #     \"\"\"\n    #     # find a match among _items\n    #     provider = self.get_provider(url)\n    #     if new or not provider:\n    #         rec = self._get_new_record_ui(url)\n    #         rec['url_re'] = \"TODO\"  # present to user and ask to edit\n    #         name = urlparse(url).netloc\n    #         self._items[name] = rec\n    #         if ui.yesno(\"Do you want to store credentials for %s\" % name):\n    #             self.store_credentials()\n    #     else:\n    #         return self._items[name]\n    #\n    # def store_credentials(self, name):\n    #     # TODO: store  self._items[name]  in appropriate (user) creds\n    #     # for later reuse\n    #     raise NotImplementedError()\n    #\n    # def _get_new_record_ui(self, url):\n    #     # TODO: should be a dialog with the fields appropriate for this particular\n    #     # type of credentials\n    #     ui.message(\"To access %s we would need credentials.\" % url)\n    #     if url in self.providers:\n    #         ui.message(\"If you don't yet have credentials, please visit %s\"\n    #                    % self.providers.get(url, 'credentials_url'))\n    #     return { 'user': ui.question(\"Username:\"),\n    #              'password': ui.password() }\n",
            "header": {
                "file": {
                    "path": "datalad/downloaders/providers.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "7eb99ca81",
                "new_commit_hash": "6c13653fb",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/downloaders/providers.py"
            },
            "plus_file": {
                "path": "b/datalad/downloaders/providers.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 18,
                            "length": 6
                        },
                        "new_range": {
                            "start": 18,
                            "length": 7
                        },
                        "section": "from six import iteritems"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "import re",
                                "type": "context"
                            },
                            {
                                "content": "from os.path import dirname, abspath, join as pathjoin",
                                "type": "context"
                            },
                            {
                                "content": "from six.moves.urllib.parse import urlparse",
                                "type": "context"
                            },
                            {
                                "content": "from collections import OrderedDict",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from .base import NoneAuthenticator, NotImplementedAuthenticator",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/providers.py",
                            "type": "statement",
                            "name": "",
                            "content": "from collections import OrderedDict",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 21,
                            "end_lineno": 21,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 199,
                            "length": 19
                        },
                        "new_range": {
                            "start": 200,
                            "length": 28
                        },
                        "section": "class Providers(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        if files is None:",
                                "type": "context"
                            },
                            {
                                "content": "            # Config files from the datalad dist",
                                "type": "context"
                            },
                            {
                                "content": "            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))",
                                "type": "context"
                            },
                            {
                                "content": "            # User config",
                                "type": "deleted"
                            },
                            {
                                "content": "            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "            # Dataset config",
                                "type": "context"
                            },
                            {
                                "content": "            dsroot = get_dataset_root(\"\")",
                                "type": "context"
                            },
                            {
                                "content": "            if dsroot is not None:",
                                "type": "context"
                            },
                            {
                                "content": "                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "            # System config",
                                "type": "context"
                            },
                            {
                                "content": "            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))",
                                "type": "deleted"
                            },
                            {
                                "content": "            if dirs.site_config_dir is not None:",
                                "type": "added"
                            },
                            {
                                "content": "                files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))",
                                "type": "added"
                            },
                            {
                                "content": "            # User config",
                                "type": "added"
                            },
                            {
                                "content": "            if dirs.user_config_dir is not None:",
                                "type": "added"
                            },
                            {
                                "content": "                files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        config.read(files)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        # We need first to load Providers and credentials",
                                "type": "context"
                            },
                            {
                                "content": "        providers = {}",
                                "type": "deleted"
                            },
                            {
                                "content": "        # Order matters, because we need to ensure that when",
                                "type": "added"
                            },
                            {
                                "content": "        # there's a conflict between configuration files declared",
                                "type": "added"
                            },
                            {
                                "content": "        # at different precedence levels (ie. dataset vs system)",
                                "type": "added"
                            },
                            {
                                "content": "        # the appropriate precedence config wins.",
                                "type": "added"
                            },
                            {
                                "content": "        providers = OrderedDict()",
                                "type": "added"
                            },
                            {
                                "content": "        credentials = {}",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "        for section in config.sections():",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/downloaders/providers.py",
                            "type": "method",
                            "name": "Providers.from_config_files",
                            "content": "class Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 179,
                            "end_lineno": 247,
                            "parent": {
                                "file_name": "datalad/downloaders/providers.py",
                                "type": "class",
                                "name": "Providers",
                                "content": "class Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 147,
                                "end_lineno": 343,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/downloaders/providers.py",
                            "type": "class",
                            "name": "Providers",
                            "content": "class Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 147,
                            "end_lineno": 343,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 296,
                            "length": 18
                        },
                        "new_range": {
                            "start": 306,
                            "length": 21
                        },
                        "section": "class Providers(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        \"\"\"Given a URL returns matching provider",
                                "type": "context"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "        nproviders = len(self._providers)",
                                "type": "context"
                            },
                            {
                                "content": "        for i in range(nproviders):",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        # Range backwards to ensure that more locally defined",
                                "type": "added"
                            },
                            {
                                "content": "        # configuration wins in conflicts between url_re",
                                "type": "added"
                            },
                            {
                                "content": "        for i in range(nproviders-1, -1, -1):",
                                "type": "added"
                            },
                            {
                                "content": "            provider = self._providers[i]",
                                "type": "context"
                            },
                            {
                                "content": "            if not provider.url_res:",
                                "type": "context"
                            },
                            {
                                "content": "                continue",
                                "type": "context"
                            },
                            {
                                "content": "            for url_re in provider.url_res:",
                                "type": "context"
                            },
                            {
                                "content": "                if re.match(url_re, url):",
                                "type": "context"
                            },
                            {
                                "content": "                    if i != 0:",
                                "type": "deleted"
                            },
                            {
                                "content": "                    #if i != 0:",
                                "type": "added"
                            },
                            {
                                "content": "                        # place it first",
                                "type": "context"
                            },
                            {
                                "content": "                        # TODO: optimize with smarter datastructures if this becomes a burden",
                                "type": "context"
                            },
                            {
                                "content": "                        del self._providers[i]",
                                "type": "deleted"
                            },
                            {
                                "content": "                        self._providers = [provider] + self._providers",
                                "type": "deleted"
                            },
                            {
                                "content": "                        assert(len(self._providers) == nproviders)",
                                "type": "deleted"
                            },
                            {
                                "content": "                        #del self._providers[i]",
                                "type": "added"
                            },
                            {
                                "content": "                        #self._providers = [provider] + self._providers",
                                "type": "added"
                            },
                            {
                                "content": "                        #assert(len(self._providers) == nproviders)",
                                "type": "added"
                            },
                            {
                                "content": "                    lgr.debug(\"Returning provider %s for url %s\", provider, url)",
                                "type": "context"
                            },
                            {
                                "content": "                    return provider",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/downloaders/providers.py",
                            "type": "method",
                            "name": "Providers.get_provider",
                            "content": "class Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 295,
                            "end_lineno": 326,
                            "parent": {
                                "file_name": "datalad/downloaders/providers.py",
                                "type": "class",
                                "name": "Providers",
                                "content": "class Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 147,
                                "end_lineno": 343,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/downloaders/providers.py",
                            "type": "class",
                            "name": "Providers",
                            "content": "class Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 147,
                            "end_lineno": 343,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for data providers\"\"\"\n\nfrom mock import patch\n\nfrom ..providers import Provider\nfrom ..providers import Providers\nfrom ..providers import HTTPDownloader\nfrom ...tests.utils import eq_\nfrom ...tests.utils import assert_in\nfrom ...tests.utils import assert_greater\nfrom ...tests.utils import assert_equal\nfrom ...tests.utils import assert_raises\n\nfrom ...support.external_versions import external_versions\n\ndef test_Providers_OnStockConfiguration():\n    providers = Providers.from_config_files()\n    provider_names = {p.name for p in providers}\n    assert_in('datalad-test-s3', provider_names)\n    assert_in('crcns', provider_names)\n    assert_greater(len(provider_names), 5)\n    # too rigid\n    #eq_(provider_names, {'crcns', 'crcns-nersc', 'hcp-http', 'hcp-s3', 'hcp-web', 'hcp-xnat', 'openfmri'})\n\n    # every provider must have url_res\n    for provider in providers:\n        assert(provider.url_res)\n\n    # and then that we didn't screw it up -- cycle few times to verify that we do not\n    # somehow remove existing providers while dealing with that \"heaplike\" list\n    for i in range(3):\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'crcns')\n\n        provider = providers.get_provider('https://portal.nersc.gov/project/crcns/download/bogus')\n        assert_equal(provider.name, 'crcns-nersc')\n\n    assert_equal(providers.needs_authentication('http://google.com'), None)\n    assert_equal(providers.needs_authentication('http://crcns.org/'), True)\n    assert_equal(providers.needs_authentication('http://openfmri.org/'), False)\n\n    providers_repr = repr(providers)\n    # should list all the providers atm\n    assert_equal(providers_repr.count('Provider('), len(providers))\n\n    # Should be a lazy evaluator unless reload is specified\n    assert(providers is Providers.from_config_files())\n    assert(providers is not Providers.from_config_files(reload=True))\n\n\ndef test_Providers_default_ones():\n    providers = Providers()  # empty one\n\n    # should return default one\n    http_provider = providers.get_provider(\"http://example.com\")\n    # which must be the same if asked for another one of http\n    http_provider2 = providers.get_provider(\"http://datalad.org\")\n    assert(http_provider is http_provider2)\n\n    # but for another protocol, we would generate a new one\n    crap_provider = providers.get_provider(\"crap://crap.crap\")\n    assert(crap_provider is not http_provider)\n    assert(isinstance(crap_provider, Provider))\n\n\ndef test_Providers_process_credential():\n    # If uknown type -- raises ValueError\n    assert_raises(ValueError, Providers._process_credential, 'cred', {'type': '_unknown_'})\n\n\ndef test_get_downloader_class():\n    url = 'http://example.com'\n\n    with patch.object(external_versions, '_versions', {'requests': 1}):\n        assert Provider._get_downloader_class(url) is HTTPDownloader\n\n    with patch.object(external_versions, '_versions', {'requests': None}):\n        with assert_raises(RuntimeError) as cmr:\n            Provider._get_downloader_class(url)\n        assert_in(\"you need 'requests'\", str(cmr.exception))",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for data providers\"\"\"\n\nimport os\nimport shutil\n\nfrom mock import patch\nfrom tempfile import mkdtemp\n\nfrom ..providers import Provider\nfrom ..providers import Providers\nfrom ..providers import HTTPDownloader\nfrom ...tests.utils import eq_\nfrom ...tests.utils import assert_in\nfrom ...tests.utils import assert_greater\nfrom ...tests.utils import assert_equal\nfrom ...tests.utils import assert_raises\n\nfrom ...support.external_versions import external_versions\nfrom ...interface.common_cfg import dirs\n\n\ndef test_Providers_OnStockConfiguration():\n    providers = Providers.from_config_files()\n    provider_names = {p.name for p in providers}\n    assert_in('datalad-test-s3', provider_names)\n    assert_in('crcns', provider_names)\n    assert_greater(len(provider_names), 5)\n    # too rigid\n    #eq_(provider_names, {'crcns', 'crcns-nersc', 'hcp-http', 'hcp-s3', 'hcp-web', 'hcp-xnat', 'openfmri'})\n\n    # every provider must have url_res\n    for provider in providers:\n        assert(provider.url_res)\n\n    # and then that we didn't screw it up -- cycle few times to verify that we do not\n    # somehow remove existing providers while dealing with that \"heaplike\" list\n    for i in range(3):\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'crcns')\n\n        provider = providers.get_provider('https://portal.nersc.gov/project/crcns/download/bogus')\n        assert_equal(provider.name, 'crcns-nersc')\n\n    assert_equal(providers.needs_authentication('http://google.com'), None)\n    assert_equal(providers.needs_authentication('http://crcns.org/'), True)\n    assert_equal(providers.needs_authentication('http://openfmri.org/'), False)\n\n    providers_repr = repr(providers)\n    # should list all the providers atm\n    assert_equal(providers_repr.count('Provider('), len(providers))\n\n    # Should be a lazy evaluator unless reload is specified\n    assert(providers is Providers.from_config_files())\n    assert(providers is not Providers.from_config_files(reload=True))\n\n\ndef test_Providers_default_ones():\n    providers = Providers()  # empty one\n\n    # should return default one\n    http_provider = providers.get_provider(\"http://example.com\")\n    # which must be the same if asked for another one of http\n    http_provider2 = providers.get_provider(\"http://datalad.org\")\n    assert(http_provider is http_provider2)\n\n    # but for another protocol, we would generate a new one\n    crap_provider = providers.get_provider(\"crap://crap.crap\")\n    assert(crap_provider is not http_provider)\n    assert(isinstance(crap_provider, Provider))\n\n\ndef test_Providers_process_credential():\n    # If uknown type -- raises ValueError\n    assert_raises(ValueError, Providers._process_credential, 'cred', {'type': '_unknown_'})\n\n\ndef test_get_downloader_class():\n    url = 'http://example.com'\n\n    with patch.object(external_versions, '_versions', {'requests': 1}):\n        assert Provider._get_downloader_class(url) is HTTPDownloader\n\n    with patch.object(external_versions, '_versions', {'requests': None}):\n        with assert_raises(RuntimeError) as cmr:\n            Provider._get_downloader_class(url)\n        assert_in(\"you need 'requests'\", str(cmr.exception))\n\n@patch.multiple(\"appdirs.AppDirs\", site_config_dir=None, user_config_dir=None)\ndef test_Providers_from_config__files():\n    \"\"\"Test configuration file precedence\n\n    Ensure that provider precedence works in the correct order:\n\n        datalad defaults < dataset defaults < system defaults < user defaults\n    \"\"\"\n\n    # Setup. This would be nice to have a context manager for, but\n    # \"with tempdir.TemporaryDirectory\" wouldn't work on Python 2.7..\n\n    # Create a fake system directory, user directory, and dataset directory\n    sysdir = mkdtemp(\"sys\", \"dataladtest\", None)\n    os.mkdir(sysdir + \"/providers/\", 0o700)\n    userdir = mkdtemp(\"user\", \"dataladtest\", None)\n    os.mkdir(userdir + \"/providers/\", 0o700)\n    dsdir = mkdtemp(\"ds\", \"dataladtest\", None)\n    os.makedirs(dsdir+ \"/.datalad/providers/\", 0o700)\n    # datalad looks for the .git directory to find the root of the\n    # dataset, not the .datalad directory.\n    os.makedirs(dsdir+ \"/.git\", 0o700)\n\n    with open(dsdir+\"/.datalad/providers/atest.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:dscrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n    with open(sysdir+\"/providers/atest2.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:syscrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n    with open(userdir+\"/providers/atest3.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:usercrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n\n    # Test the default, this is an arbitrary provider used from another\n    # test\n    providers = Providers.from_config_files(reload=True)\n    provider = providers.get_provider('https://crcns.org/data....')\n    assert_equal(provider.name, 'crcns')\n\n    # Test that the dataset provider overrides the datalad\n    # default\n    os.chdir(dsdir)\n    providers = Providers.from_config_files(reload=True)\n    provider = providers.get_provider('https://crcns.org/data....')\n    assert_equal(provider.name, 'dscrcns')\n\n    # Test that the system defaults take precedence over the dataset\n    # defaults (we're still within the dsdir)\n    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=None):\n        providers = Providers.from_config_files(reload=True)\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'syscrcns')\n\n    # Test that the user defaults take precedence over the system\n    # defaults\n    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=userdir):\n        providers = Providers.from_config_files(reload=True)\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'usercrcns')\n\n    # Cleanup\n    shutil.rmtree(sysdir)\n    shutil.rmtree(userdir)\n    shutil.rmtree(dsdir)\n",
            "header": {
                "file": {
                    "path": "datalad/downloaders/tests/test_providers.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "d7648f913",
                "new_commit_hash": "4b166ab45",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/downloaders/tests/test_providers.py"
            },
            "plus_file": {
                "path": "b/datalad/downloaders/tests/test_providers.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 8,
                            "length": 7
                        },
                        "new_range": {
                            "start": 8,
                            "length": 11
                        },
                        "section": ""
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##",
                                "type": "context"
                            },
                            {
                                "content": "\"\"\"Tests for data providers\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "import os",
                                "type": "added"
                            },
                            {
                                "content": "import shutil",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "from mock import patch",
                                "type": "context"
                            },
                            {
                                "content": "from tempfile import mkdtemp",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from ..providers import Provider",
                                "type": "context"
                            },
                            {
                                "content": "from ..providers import Providers",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_providers.py",
                            "type": "import",
                            "name": "os",
                            "content": "import os",
                            "ast_type_str": "Import",
                            "start_lineno": 11,
                            "end_lineno": 11,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/downloaders/tests/test_providers.py",
                            "type": "statement",
                            "name": "",
                            "content": "from tempfile import mkdtemp",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 15,
                            "end_lineno": 15,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/downloaders/tests/test_providers.py",
                            "type": "import",
                            "name": "shutil",
                            "content": "import shutil",
                            "ast_type_str": "Import",
                            "start_lineno": 12,
                            "end_lineno": 12,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 20,
                            "length": 6
                        },
                        "new_range": {
                            "start": 24,
                            "length": 8
                        },
                        "section": "from ...tests.utils import assert_equal"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "from ...tests.utils import assert_raises",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from ...support.external_versions import external_versions",
                                "type": "context"
                            },
                            {
                                "content": "from ...interface.common_cfg import dirs",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_Providers_OnStockConfiguration():",
                                "type": "context"
                            },
                            {
                                "content": "    providers = Providers.from_config_files()",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_providers.py",
                            "type": "statement",
                            "name": "",
                            "content": "from ...interface.common_cfg import dirs",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 27,
                            "end_lineno": 27,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 85,
                            "length": 4
                        },
                        "new_range": {
                            "start": 91,
                            "length": 75
                        },
                        "section": "def test_get_downloader_class():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    with patch.object(external_versions, '_versions', {'requests': None}):",
                                "type": "context"
                            },
                            {
                                "content": "        with assert_raises(RuntimeError) as cmr:",
                                "type": "context"
                            },
                            {
                                "content": "            Provider._get_downloader_class(url)",
                                "type": "context"
                            },
                            {
                                "content": "        assert_in(\"you need 'requests'\", str(cmr.exception))",
                                "type": "deleted"
                            },
                            {
                                "content": "No newline at end of file",
                                "type": "note"
                            },
                            {
                                "content": "        assert_in(\"you need 'requests'\", str(cmr.exception))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "@patch.multiple(\"appdirs.AppDirs\", site_config_dir=None, user_config_dir=None)",
                                "type": "added"
                            },
                            {
                                "content": "def test_Providers_from_config__files():",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Test configuration file precedence",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Ensure that provider precedence works in the correct order:",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "        datalad defaults < dataset defaults < system defaults < user defaults",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Setup. This would be nice to have a context manager for, but",
                                "type": "added"
                            },
                            {
                                "content": "    # \"with tempdir.TemporaryDirectory\" wouldn't work on Python 2.7..",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Create a fake system directory, user directory, and dataset directory",
                                "type": "added"
                            },
                            {
                                "content": "    sysdir = mkdtemp(\"sys\", \"dataladtest\", None)",
                                "type": "added"
                            },
                            {
                                "content": "    os.mkdir(sysdir + \"/providers/\", 0o700)",
                                "type": "added"
                            },
                            {
                                "content": "    userdir = mkdtemp(\"user\", \"dataladtest\", None)",
                                "type": "added"
                            },
                            {
                                "content": "    os.mkdir(userdir + \"/providers/\", 0o700)",
                                "type": "added"
                            },
                            {
                                "content": "    dsdir = mkdtemp(\"ds\", \"dataladtest\", None)",
                                "type": "added"
                            },
                            {
                                "content": "    os.makedirs(dsdir+ \"/.datalad/providers/\", 0o700)",
                                "type": "added"
                            },
                            {
                                "content": "    # datalad looks for the .git directory to find the root of the",
                                "type": "added"
                            },
                            {
                                "content": "    # dataset, not the .datalad directory.",
                                "type": "added"
                            },
                            {
                                "content": "    os.makedirs(dsdir+ \"/.git\", 0o700)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    with open(dsdir+\"/.datalad/providers/atest.cfg\", \"w\") as f:",
                                "type": "added"
                            },
                            {
                                "content": "        f.write(\"\"\"[provider:dscrcns]",
                                "type": "added"
                            },
                            {
                                "content": "url_re = https?://crcns\\.org/.*",
                                "type": "added"
                            },
                            {
                                "content": "authentication_type = none",
                                "type": "added"
                            },
                            {
                                "content": "        \"\"\")",
                                "type": "added"
                            },
                            {
                                "content": "    with open(sysdir+\"/providers/atest2.cfg\", \"w\") as f:",
                                "type": "added"
                            },
                            {
                                "content": "        f.write(\"\"\"[provider:syscrcns]",
                                "type": "added"
                            },
                            {
                                "content": "url_re = https?://crcns\\.org/.*",
                                "type": "added"
                            },
                            {
                                "content": "authentication_type = none",
                                "type": "added"
                            },
                            {
                                "content": "        \"\"\")",
                                "type": "added"
                            },
                            {
                                "content": "    with open(userdir+\"/providers/atest3.cfg\", \"w\") as f:",
                                "type": "added"
                            },
                            {
                                "content": "        f.write(\"\"\"[provider:usercrcns]",
                                "type": "added"
                            },
                            {
                                "content": "url_re = https?://crcns\\.org/.*",
                                "type": "added"
                            },
                            {
                                "content": "authentication_type = none",
                                "type": "added"
                            },
                            {
                                "content": "        \"\"\")",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Test the default, this is an arbitrary provider used from another",
                                "type": "added"
                            },
                            {
                                "content": "    # test",
                                "type": "added"
                            },
                            {
                                "content": "    providers = Providers.from_config_files(reload=True)",
                                "type": "added"
                            },
                            {
                                "content": "    provider = providers.get_provider('https://crcns.org/data....')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_equal(provider.name, 'crcns')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Test that the dataset provider overrides the datalad",
                                "type": "added"
                            },
                            {
                                "content": "    # default",
                                "type": "added"
                            },
                            {
                                "content": "    os.chdir(dsdir)",
                                "type": "added"
                            },
                            {
                                "content": "    providers = Providers.from_config_files(reload=True)",
                                "type": "added"
                            },
                            {
                                "content": "    provider = providers.get_provider('https://crcns.org/data....')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_equal(provider.name, 'dscrcns')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Test that the system defaults take precedence over the dataset",
                                "type": "added"
                            },
                            {
                                "content": "    # defaults (we're still within the dsdir)",
                                "type": "added"
                            },
                            {
                                "content": "    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=None):",
                                "type": "added"
                            },
                            {
                                "content": "        providers = Providers.from_config_files(reload=True)",
                                "type": "added"
                            },
                            {
                                "content": "        provider = providers.get_provider('https://crcns.org/data....')",
                                "type": "added"
                            },
                            {
                                "content": "        assert_equal(provider.name, 'syscrcns')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Test that the user defaults take precedence over the system",
                                "type": "added"
                            },
                            {
                                "content": "    # defaults",
                                "type": "added"
                            },
                            {
                                "content": "    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=userdir):",
                                "type": "added"
                            },
                            {
                                "content": "        providers = Providers.from_config_files(reload=True)",
                                "type": "added"
                            },
                            {
                                "content": "        provider = providers.get_provider('https://crcns.org/data....')",
                                "type": "added"
                            },
                            {
                                "content": "        assert_equal(provider.name, 'usercrcns')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Cleanup",
                                "type": "added"
                            },
                            {
                                "content": "    shutil.rmtree(sysdir)",
                                "type": "added"
                            },
                            {
                                "content": "    shutil.rmtree(userdir)",
                                "type": "added"
                            },
                            {
                                "content": "    shutil.rmtree(dsdir)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_providers.py",
                            "type": "function",
                            "name": "test_get_downloader_class",
                            "content": "def test_get_downloader_class():\n    url = 'http://example.com'\n\n    with patch.object(external_versions, '_versions', {'requests': 1}):\n        assert Provider._get_downloader_class(url) is HTTPDownloader\n\n    with patch.object(external_versions, '_versions', {'requests': None}):\n        with assert_raises(RuntimeError) as cmr:\n            Provider._get_downloader_class(url)\n        assert_in(\"you need 'requests'\", str(cmr.exception))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 79,
                            "end_lineno": 88,
                            "parent": null
                        }
                    ],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_providers.py",
                            "type": "function",
                            "name": "test_Providers_from_config__files",
                            "content": "def test_Providers_from_config__files():\n    \"\"\"Test configuration file precedence\n\n    Ensure that provider precedence works in the correct order:\n\n        datalad defaults < dataset defaults < system defaults < user defaults\n    \"\"\"\n\n    # Setup. This would be nice to have a context manager for, but\n    # \"with tempdir.TemporaryDirectory\" wouldn't work on Python 2.7..\n\n    # Create a fake system directory, user directory, and dataset directory\n    sysdir = mkdtemp(\"sys\", \"dataladtest\", None)\n    os.mkdir(sysdir + \"/providers/\", 0o700)\n    userdir = mkdtemp(\"user\", \"dataladtest\", None)\n    os.mkdir(userdir + \"/providers/\", 0o700)\n    dsdir = mkdtemp(\"ds\", \"dataladtest\", None)\n    os.makedirs(dsdir+ \"/.datalad/providers/\", 0o700)\n    # datalad looks for the .git directory to find the root of the\n    # dataset, not the .datalad directory.\n    os.makedirs(dsdir+ \"/.git\", 0o700)\n\n    with open(dsdir+\"/.datalad/providers/atest.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:dscrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n    with open(sysdir+\"/providers/atest2.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:syscrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n    with open(userdir+\"/providers/atest3.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:usercrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n\n    # Test the default, this is an arbitrary provider used from another\n    # test\n    providers = Providers.from_config_files(reload=True)\n    provider = providers.get_provider('https://crcns.org/data....')\n    assert_equal(provider.name, 'crcns')\n\n    # Test that the dataset provider overrides the datalad\n    # default\n    os.chdir(dsdir)\n    providers = Providers.from_config_files(reload=True)\n    provider = providers.get_provider('https://crcns.org/data....')\n    assert_equal(provider.name, 'dscrcns')\n\n    # Test that the system defaults take precedence over the dataset\n    # defaults (we're still within the dsdir)\n    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=None):\n        providers = Providers.from_config_files(reload=True)\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'syscrcns')\n\n    # Test that the user defaults take precedence over the system\n    # defaults\n    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=userdir):\n        providers = Providers.from_config_files(reload=True)\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'usercrcns')\n\n    # Cleanup\n    shutil.rmtree(sysdir)\n    shutil.rmtree(userdir)\n    shutil.rmtree(dsdir)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 97,
                            "end_lineno": 165,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "26df45e6833f5ac32af463cae8c3cb6bd86971da^",
    "new_commit_hash": "26df45e6833f5ac32af463cae8c3cb6bd86971da",
    "commit_message": "Add tests for provider configuration precedence",
    "commit_date": "2018-06-07T11:13:32-04:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "26df45e6833f5ac32af463cae8c3cb6bd86971da",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for data providers\"\"\"\n\nimport os\nimport shutil\n\nfrom mock import patch\nfrom tempfile import mkdtemp\n\nfrom datalad.downloaders.providers import Provider\nfrom datalad.downloaders.providers import Providers\nfrom datalad.downloaders.providers import HTTPDownloader\nfrom datalad.tests.utils import eq_\nfrom datalad.tests.utils import assert_in\nfrom datalad.tests.utils import assert_greater\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_raises\n\nfrom datalad.support.external_versions import external_versions\nfrom datalad.interface.common_cfg import dirs\n\n\ndef test_Providers_OnStockConfiguration():\n    providers = Providers.from_config_files()\n    provider_names = {p.name for p in providers}\n    assert_in('datalad-test-s3', provider_names)\n    assert_in('crcns', provider_names)\n    assert_greater(len(provider_names), 5)\n    # too rigid\n    #eq_(provider_names, {'crcns', 'crcns-nersc', 'hcp-http', 'hcp-s3', 'hcp-web', 'hcp-xnat', 'openfmri'})\n\n    # every provider must have url_res\n    for provider in providers:\n        assert(provider.url_res)\n\n    # and then that we didn't screw it up -- cycle few times to verify that we do not\n    # somehow remove existing providers while dealing with that \"heaplike\" list\n    for i in range(3):\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'crcns')\n\n        provider = providers.get_provider('https://portal.nersc.gov/project/crcns/download/bogus')\n        assert_equal(provider.name, 'crcns-nersc')\n\n    assert_equal(providers.needs_authentication('http://google.com'), None)\n    assert_equal(providers.needs_authentication('http://crcns.org/'), True)\n    assert_equal(providers.needs_authentication('http://openfmri.org/'), False)\n\n    providers_repr = repr(providers)\n    # should list all the providers atm\n    assert_equal(providers_repr.count('Provider('), len(providers))\n\n    # Should be a lazy evaluator unless reload is specified\n    assert(providers is Providers.from_config_files())\n    assert(providers is not Providers.from_config_files(reload=True))\n\n\ndef test_Providers_default_ones():\n    providers = Providers()  # empty one\n\n    # should return default one\n    http_provider = providers.get_provider(\"http://example.com\")\n    # which must be the same if asked for another one of http\n    http_provider2 = providers.get_provider(\"http://datalad.org\")\n    assert(http_provider is http_provider2)\n\n    # but for another protocol, we would generate a new one\n    crap_provider = providers.get_provider(\"crap://crap.crap\")\n    assert(crap_provider is not http_provider)\n    assert(isinstance(crap_provider, Provider))\n\n\ndef test_Providers_process_credential():\n    # If uknown type -- raises ValueError\n    assert_raises(ValueError, Providers._process_credential, 'cred', {'type': '_unknown_'})\n\n\ndef test_get_downloader_class():\n    url = 'http://example.com'\n\n    with patch.object(external_versions, '_versions', {'requests': 1}):\n        assert Provider._get_downloader_class(url) is HTTPDownloader\n\n    with patch.object(external_versions, '_versions', {'requests': None}):\n        with assert_raises(RuntimeError) as cmr:\n            Provider._get_downloader_class(url)\n        assert_in(\"you need 'requests'\", str(cmr.exception))\n\n@patch.multiple(\"appdirs.AppDirs\", site_config_dir=None, user_config_dir=None)\ndef test_Providers_from_config__files():\n    \"\"\"Test configuration file precedence\n\n    Ensure that provider precedence works in the correct order:\n\n        datalad defaults < dataset defaults < system defaults < user defaults\n    \"\"\"\n\n    # Setup. This would be nice to have a context manager for, but\n    # \"with tempdir.TemporaryDirectory\" wouldn't work on Python 2.7..\n\n    # Create a fake system directory, user directory, and dataset directory\n    sysdir = mkdtemp(\"sys\", \"dataladtest\", None)\n    os.mkdir(sysdir + \"/providers/\", 0o700)\n    userdir = mkdtemp(\"user\", \"dataladtest\", None)\n    os.mkdir(userdir + \"/providers/\", 0o700)\n    dsdir = mkdtemp(\"ds\", \"dataladtest\", None)\n    os.makedirs(dsdir+ \"/.datalad/providers/\", 0o700)\n    # datalad looks for the .git directory to find the root of the\n    # dataset, not the .datalad directory.\n    os.makedirs(dsdir+ \"/.git\", 0o700)\n\n    with open(dsdir+\"/.datalad/providers/atest.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:dscrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n    with open(sysdir+\"/providers/atest2.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:syscrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n    with open(userdir+\"/providers/atest3.cfg\", \"w\") as f:\n        f.write(\"\"\"[provider:usercrcns]\nurl_re = https?://crcns\\.org/.*\nauthentication_type = none\n        \"\"\")\n\n    # Test the default, this is an arbitrary provider used from another\n    # test\n    providers = Providers.from_config_files(reload=True)\n    provider = providers.get_provider('https://crcns.org/data....')\n    assert_equal(provider.name, 'crcns')\n\n    # Test that the dataset provider overrides the datalad\n    # default\n    os.chdir(dsdir)\n    providers = Providers.from_config_files(reload=True)\n    provider = providers.get_provider('https://crcns.org/data....')\n    assert_equal(provider.name, 'dscrcns')\n\n    # Test that the system defaults take precedence over the dataset\n    # defaults (we're still within the dsdir)\n    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=None):\n        providers = Providers.from_config_files(reload=True)\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'syscrcns')\n\n    # Test that the user defaults take precedence over the system\n    # defaults\n    with patch.multiple(\"appdirs.AppDirs\", site_config_dir=sysdir, user_config_dir=userdir):\n        providers = Providers.from_config_files(reload=True)\n        provider = providers.get_provider('https://crcns.org/data....')\n        assert_equal(provider.name, 'usercrcns')\n\n    # Cleanup\n    shutil.rmtree(sysdir)\n    shutil.rmtree(userdir)\n    shutil.rmtree(dsdir)\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.10.0.rc5.dev18\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 130ms\nInstalled 25 packages in 5.79s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Y7bVr/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Y7bVr/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Y7bVr/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Y7bVr/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Y7bVr/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpVq3PmI/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpVq3PmI/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpVq3PmI/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpVq3PmI/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpVq3PmI/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_26df45e6833f5ac32af463cae8c3cb6bd86971da/datalad/__init__.py\", line 23, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_26df45e6833f5ac32af463cae8c3cb6bd86971da/datalad/cmd.py\", line 28, in <module>\n    from .support import path as op\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_26df45e6833f5ac32af463cae8c3cb6bd86971da/datalad/support/path.py\", line 18, in <module>\n    from ..utils import assure_bytes\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_26df45e6833f5ac32af463cae8c3cb6bd86971da/datalad/utils.py\", line 25, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 258ms\nInstalled 26 packages in 3.78s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 57 packages in 1.29s\nPrepared 1 package in 1.36s\nInstalled 50 packages in 62ms\n + appdirs==1.4.4\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + cryptography==44.0.0\n + datalad==0.10.0rc5 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_26df45e6833f5ac32af463cae8c3cb6bd86971da)\n + deprecated==1.2.15\n + exifread==3.0.0\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + idna==3.10\n + importlib-resources==5.12.0\n + iso8601==2.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack==1.0.5\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.12\n + pillow==9.5.0\n + pycparser==2.21\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + requests==2.31.0\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.1\n + soupsieve==2.4.1\n + tqdm==4.67.1\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\nResolved 89 packages in 188ms\nInstalled 30 packages in 143ms\n + alabaster==0.7.13\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.14.0\n + build==1.1.1\n + distlib==0.3.9\n + docutils==0.19\n + filelock==3.12.2\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.9.16\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + platformdirs==4.0.0\n + pygments==2.17.2\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n + tabulate==0.9.0\n + virtualenv==20.26.6\n",
    "new_commit_res_code": 0,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_26df45e6833f5ac32af463cae8c3cb6bd86971da\nplugins: cov-4.1.0\ncollected 5 items\n\nr2e_tests/test_1.py .....                                                [100%]\n\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_Providers_OnStockConfiguration\nPASSED r2e_tests/test_1.py::test_Providers_default_ones\nPASSED r2e_tests/test_1.py::test_Providers_process_credential\nPASSED r2e_tests/test_1.py::test_get_downloader_class\nPASSED r2e_tests/test_1.py::test_Providers_from_config__files\n============================== 5 passed in 1.13s ===============================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_26df45e6833f5ac32af463cae8c3cb6bd86971da\nplugins: cov-4.1.0\ncollected 5 items\n\nr2e_tests/test_1.py ....F                                                [100%]\n\n=================================== FAILURES ===================================\n______________________ test_Providers_from_config__files _______________________\n\n    @patch.multiple(\"appdirs.AppDirs\", site_config_dir=None, user_config_dir=None)\n    def test_Providers_from_config__files():\n        \"\"\"Test configuration file precedence\n    \n        Ensure that provider precedence works in the correct order:\n    \n            datalad defaults < dataset defaults < system defaults < user defaults\n        \"\"\"\n    \n        # Setup. This would be nice to have a context manager for, but\n        # \"with tempdir.TemporaryDirectory\" wouldn't work on Python 2.7..\n    \n        # Create a fake system directory, user directory, and dataset directory\n        sysdir = mkdtemp(\"sys\", \"dataladtest\", None)\n        os.mkdir(sysdir + \"/providers/\", 0o700)\n        userdir = mkdtemp(\"user\", \"dataladtest\", None)\n        os.mkdir(userdir + \"/providers/\", 0o700)\n        dsdir = mkdtemp(\"ds\", \"dataladtest\", None)\n        os.makedirs(dsdir+ \"/.datalad/providers/\", 0o700)\n        # datalad looks for the .git directory to find the root of the\n        # dataset, not the .datalad directory.\n        os.makedirs(dsdir+ \"/.git\", 0o700)\n    \n        with open(dsdir+\"/.datalad/providers/atest.cfg\", \"w\") as f:\n            f.write(\"\"\"[provider:dscrcns]\n    url_re = https?://crcns\\.org/.*\n    authentication_type = none\n            \"\"\")\n        with open(sysdir+\"/providers/atest2.cfg\", \"w\") as f:\n            f.write(\"\"\"[provider:syscrcns]\n    url_re = https?://crcns\\.org/.*\n    authentication_type = none\n            \"\"\")\n        with open(userdir+\"/providers/atest3.cfg\", \"w\") as f:\n            f.write(\"\"\"[provider:usercrcns]\n    url_re = https?://crcns\\.org/.*\n    authentication_type = none\n            \"\"\")\n    \n        # Test the default, this is an arbitrary provider used from another\n        # test\n>       providers = Providers.from_config_files(reload=True)\n\nr2e_tests/test_1.py:137: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/downloaders/providers.py:203: in from_config_files\n    files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = None, p = ('providers', '*.cfg')\n\n    def join(a, *p):\n        \"\"\"Join two or more pathname components, inserting '/' as needed.\n        If any component is an absolute path, all previous path components\n        will be discarded.  An empty last part will result in a path that\n        ends with a separator.\"\"\"\n>       a = os.fspath(a)\nE       TypeError: expected str, bytes or os.PathLike object, not NoneType\n\n../../../../.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/posixpath.py:80: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_Providers_OnStockConfiguration\nPASSED r2e_tests/test_1.py::test_Providers_default_ones\nPASSED r2e_tests/test_1.py::test_Providers_process_credential\nPASSED r2e_tests/test_1.py::test_get_downloader_class\nFAILED r2e_tests/test_1.py::test_Providers_from_config__files - TypeError: ex...\n========================= 1 failed, 4 passed in 0.51s ==========================\n",
    "old_commit_res_stderr": ""
}
datalad/downloaders/providers.py datalad/downloaders/tests/test_providers.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** TypeError When Configuration Directories Are Undefined During Provider Loading

**Description:**
When the user or site configuration directories are not set, attempting to load provider configuration files causes a `TypeError`. This error occurs because the system tries to join path components where one of the directories is `None`, leading to an unexpected failure.

**Example Code:**
```python
from datalad.downloaders.providers import Providers
from appdirs import AppDirs
from glob import glob
from os.path import join

# Simulate undefined configuration directories
AppDirs.user_config_dir = None
AppDirs.site_config_dir = None

# Attempt to load provider configurations
providers = Providers.from_config_files()
provider = providers.get_provider('https://crcns.org/data....')
```

**Expected Behavior:**
Provider configuration loading should handle `None` values for configuration directories gracefully by skipping them. The system should load available configuration files without raising errors, ensuring that provider precedence is maintained correctly.

**Actual Behavior:**
A `TypeError` is raised with the message:
```
TypeError: expected str, bytes or os.PathLike object, not NoneType
```
This error stops the provider configurations from being loaded when the configuration directories are not defined.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/downloaders/providers.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Data providers - bind downloaders and credentials together
"""

import os

from glob import glob
from logging import getLogger
from six import iteritems

import re
from os.path import dirname, abspath, join as pathjoin
from six.moves.urllib.parse import urlparse

from .base import NoneAuthenticator, NotImplementedAuthenticator

from .http import HTMLFormAuthenticator, HTTPBasicAuthAuthenticator, HTTPDigestAuthAuthenticator
from .http import HTTPDownloader
from .s3 import S3Authenticator, S3Downloader
from ..support.configparserinc import SafeConfigParserWithIncludes
from ..support.external_versions import external_versions
from ..utils import assure_list_from_str
from ..utils import auto_repr
from ..utils import get_dataset_root

from ..interface.common_cfg import dirs

lgr = getLogger('datalad.downloaders.providers')

# dict to bind authentication_type's to authenticator classes
# parameters will be fetched from config file itself
AUTHENTICATION_TYPES = {
    'html_form': HTMLFormAuthenticator,
    'http_auth': HTTPBasicAuthAuthenticator,
    'http_basic_auth': HTTPBasicAuthAuthenticator,
    'http_digest_auth': HTTPDigestAuthAuthenticator,
    'aws-s3': S3Authenticator,  # TODO: check if having '-' is kosher
    'nda-s3': S3Authenticator,
    'xnat': NotImplementedAuthenticator,
    'none': NoneAuthenticator,
}

from .credentials import CREDENTIAL_TYPES

def resolve_url_to_name(d, url):
    """Given a directory (e.g. of SiteInformation._items or Credential._items)
    go through url_re and find the corresponding item and returns its key (i.e. name)
    """

    for k, rec in iteritems(d):
        for url_re in rec.get('url_re', '').split('\n'):
            if url_re:
                if re.search(url_re, url):
                    return k
    return None


@auto_repr
class Provider(object):
    """Class to bring together url_res, credential, and authenticator
    """
    # TODO: we might need a lazy loading of the submodules which would provide
    # specific downloaders while importing needed Python modules "on demand"
    DOWNLOADERS = {
        'http': {'class': HTTPDownloader, 'externals': {'requests'}},
        'https': {'class': HTTPDownloader, 'externals': {'requests'}},
        'ftp': {'class': HTTPDownloader, 'externals': {'requests', 'boto'}},
        's3': {'class': S3Downloader, 'externals': {'boto'}}
        # ... TODO
    }

    def __init__(self, name, url_res, credential=None, authenticator=None,
                 downloader=None):
        """
        Parameters
        ----------
        name: str
        url_res: list of str
           Regular expressions
        credential: Credential, optional
        authenticator: Authenticator, optional
        downloader: Downloader, optional

        """
        self.name = name
        self.url_res = assure_list_from_str(url_res)
        self.credential = credential
        self.authenticator = authenticator
        self._downloader = downloader

    @property
    def downloader(self):
        return self._downloader

    @staticmethod
    def get_scheme_from_url(url):
        """Given a URL return scheme to decide which downloader class to use
        """
        url_split = urlparse(url)
        return url_split.scheme  # , url_split.netloc)

    @classmethod
    def _get_downloader_class(cls, url):
        key = cls.get_scheme_from_url(url)
        if key in cls.DOWNLOADERS:
            entry = cls.DOWNLOADERS[key]
            klass = entry['class']
            for ext in entry.get('externals', set()):
                if external_versions[ext] is None:
                    raise RuntimeError(
                        "For using %s downloader, you need '%s' dependency "
                        "which seems to be missing" % (klass, ext)
                    )
            return klass
        else:
            raise ValueError("Do not know how to handle url %s for scheme %s. Known: %s"
                             % (url, key, cls.DOWNLOADERS.keys()))

    def get_downloader(self, url, **kwargs):
        """Assigns proper downloader given the URL

        If one is known -- verifies its appropriateness for the given url.
        ATM we do not support multiple types of downloaders per single provider
        """
        if self._downloader is None:
            # we need to create a new one
            Downloader = self._get_downloader_class(url)
            # we might need to provide it with credentials and authenticator
            # Let's do via kwargs so we could accomodate cases when downloader does not necessarily
            # cares about those... duck typing or what it is in action
            kwargs = kwargs.copy()
            if self.credential:
                kwargs['credential'] = self.credential
            if self.authenticator:
                kwargs['authenticator'] = self.authenticator
            self._downloader = Downloader(**kwargs)
        return self._downloader


class Providers(object):
    """

    So we could provide handling for URLs with corresponding credentials
    and specific (reusable) downloader.  Internally it contains
    Providers and interfaces them based on a given URL.  Each provider
    in turn takes care about associated with it Downloader.
    """

    _DEFAULT_PROVIDERS = None

    def __init__(self, providers=None):
        """
        """
        self._providers = providers or []
        # a set of providers to handle connections without authentication.
        # Will be setup one per each protocol schema
        self._default_providers = {}

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, "" if not self._providers else repr(self._providers))

    def __len__(self):
        return len(self._providers)

    def __getitem__(self, index):
        return self._providers[index]

    def __iter__(self):
        return self._providers.__iter__()

    @classmethod
    def from_config_files(cls, files=None, reload=False):
        """Loads information about related/possible websites requiring authentication from:

        - datalad/downloaders/configs/*.cfg files provided by the codebase
        - current dataset .datalad/providers/
        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)
        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)

        For sample configs files see datalad/downloaders/configs/providers.cfg

        If files is None, loading is cached between calls.  Specify reload=True to force
        reloading of files from the filesystem.  The class method reset_default_providers
        can also be called to reset the cached providers.
        """
        # lazy part
        if files is None and cls._DEFAULT_PROVIDERS and not reload:
            return cls._DEFAULT_PROVIDERS

        config = SafeConfigParserWithIncludes()
        files_orig = files
        if files is None:
            # Config files from the datalad dist
            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))
            # User config
            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))

            # Dataset config
            dsroot = get_dataset_root("")
            if dsroot is not None:
                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))
            # System config
            files.extend(glob(pathjoin(dirs.site_config_dir, "providers", "*.cfg")))
        config.read(files)

        # We need first to load Providers and credentials
        providers = {}
        credentials = {}

        for section in config.sections():
            if ':' in section:
                type_, name = section.split(':', 1)
                assert type_ in {'provider', 'credential'}, "we know only providers and credentials, got type %s" % type_
                items = {
                    o: config.get(section, o) for o in config.options(section)
                }
                # side-effect -- items get poped
                locals().get(type_ + "s")[name] = getattr(
                    cls, '_process_' + type_)(name, items)
                if len(items):
                    raise ValueError("Unprocessed fields left for %s: %s" % (name, str(items)))
            else:
                lgr.warning("Do not know how to treat section %s here" % section)

        # link credentials into providers
        lgr.debug("Assigning credentials into %d providers" % len(providers))
        for provider in providers.values():
            if provider.credential:
                if provider.credential not in credentials:
                    raise ValueError("Unknown credential %s. Known are: %s"
                                     % (provider.credential, ", ".join(credentials.keys())))
                provider.credential = credentials[provider.credential]

        providers = Providers(list(providers.values()))

        if files_orig is None:
            # Store providers for lazy access
            cls._DEFAULT_PROVIDERS = providers

        return providers

    @classmethod
    def reset_default_providers(cls):
        """Resets to None memoized by from_config_files providers
        """
        cls._DEFAULT_PROVIDERS = None

    @classmethod
    def _process_provider(cls, name, items):
        """Process a dictionary specifying the provider and output the Provider instance
        """
        assert 'authentication_type' in items, "Must have authentication_type specified"

        auth_type = items.pop('authentication_type')
        if auth_type not in AUTHENTICATION_TYPES:
            raise ValueError("Unknown authentication_type=%s. Known are: %s"
                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))

        if auth_type != 'none':
            authenticator = AUTHENTICATION_TYPES[auth_type](
                # Extract all the fields as keyword arguments
                **{k[len(auth_type) + 1:]: items.pop(k)
                   for k in list(items.keys())
                   if k.startswith(auth_type + "_")}
            )
        else:
            authenticator = None

        # bringing url_re to "standard" format of a list and populating _providers_ordered
        url_res = assure_list_from_str(items.pop('url_re'))
        assert url_res, "current implementation relies on having url_re defined"

        credential = items.pop('credential', None)

        # credential instance will be assigned later after all of them are loaded
        return Provider(name=name, url_res=url_res, authenticator=authenticator,
                        credential=credential)

    @classmethod
    def _process_credential(cls, name, items):
        assert 'type' in items, "Credential must specify type.  Missing in %s" % name
        cred_type = items.pop('type')
        if not cred_type in CREDENTIAL_TYPES:
            raise ValueError("I do not know type %s credential. Known: %s"
                             % (cred_type, CREDENTIAL_TYPES.keys()))
        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))

    def get_provider(self, url, only_nondefault=False):
        """Given a URL returns matching provider
        """
        nproviders = len(self._providers)
        for i in range(nproviders):
            provider = self._providers[i]
            if not provider.url_res:
                continue
            for url_re in provider.url_res:
                if re.match(url_re, url):
                    if i != 0:
                        # place it first
                        # TODO: optimize with smarter datastructures if this becomes a burden
                        del self._providers[i]
                        self._providers = [provider] + self._providers
                        assert(len(self._providers) == nproviders)
                    lgr.debug("Returning provider %s for url %s", provider, url)
                    return provider

        if only_nondefault:
            return None

        # None matched -- so we should get a default one per each of used
        # protocols
        scheme = Provider.get_scheme_from_url(url)
        if scheme not in self._default_providers:
            lgr.debug("Initializing default provider for %s" % scheme)
            self._default_providers[scheme] = Provider(name="", url_res=["%s://.*" % scheme])
        provider = self._default_providers[scheme]
        lgr.debug("No dedicated provider, returning default one for %s: %s",
                  scheme, provider)
        return provider

    # TODO: avoid duplication somehow ;)
    # Sugarings to get easier access to downloaders
    def download(self, url, *args, **kwargs):
        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)

    def fetch(self, url, *args, **kwargs):
        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)

    def get_status(self, url, *args, **kwargs):
        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)

    def needs_authentication(self, url):
        provider = self.get_provider(url, only_nondefault=True)
        if provider is None:
            return None
        return provider.authenticator is not None


    # # TODO: UNUSED?
    # def get_credentials(self, url, new=False):
    #     """Ask user to enter credentials for a provider matching url
    #     """
    #     # find a match among _items
    #     provider = self.get_provider(url)
    #     if new or not provider:
    #         rec = self._get_new_record_ui(url)
    #         rec['url_re'] = "TODO"  # present to user and ask to edit
    #         name = urlparse(url).netloc
    #         self._items[name] = rec
    #         if ui.yesno("Do you want to store credentials for %s" % name):
    #             self.store_credentials()
    #     else:
    #         return self._items[name]
    #
    # def store_credentials(self, name):
    #     # TODO: store  self._items[name]  in appropriate (user) creds
    #     # for later reuse
    #     raise NotImplementedError()
    #
    # def _get_new_record_ui(self, url):
    #     # TODO: should be a dialog with the fields appropriate for this particular
    #     # type of credentials
    #     ui.message("To access %s we would need credentials." % url)
    #     if url in self.providers:
    #         ui.message("If you don't yet have credentials, please visit %s"
    #                    % self.providers.get(url, 'credentials_url'))
    #     return { 'user': ui.question("Username:"),
    #              'password': ui.password() }

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** TypeError When Configuration Directories Are Undefined During Provider Loading

**Description:**
When the user or site configuration directories are not set, attempting to load provider configuration files causes a `TypeError`. This error occurs because the system tries to join path components where one of the directories is `None`, leading to an unexpected failure.

**Example Code:**
```python
from datalad.downloaders.providers import Providers
from appdirs import AppDirs
from glob import glob
from os.path import join

# Simulate undefined configuration directories
AppDirs.user_config_dir = None
AppDirs.site_config_dir = None

# Attempt to load provider configurations
providers = Providers.from_config_files()
provider = providers.get_provider('https://crcns.org/data....')
```

**Expected Behavior:**
Provider configuration loading should handle `None` values for configuration directories gracefully by skipping them. The system should load available configuration files without raising errors, ensuring that provider precedence is maintained correctly.

**Actual Behavior:**
A `TypeError` is raised with the message:
```
TypeError: expected str, bytes or os.PathLike object, not NoneType
```
This error stops the provider configurations from being loaded when the configuration directories are not defined.
[/ISSUE]
{
    "test_Providers_OnStockConfiguration": "PASSED",
    "test_Providers_default_ones": "PASSED",
    "test_Providers_process_credential": "PASSED",
    "test_get_downloader_class": "PASSED",
    "test_Providers_from_config__files": "PASSED"
}
diff --git a/datalad/downloaders/providers.py b/datalad/downloaders/providers.py
index 7eb99ca81..6c13653fb 100644
--- a/datalad/downloaders/providers.py
+++ b/datalad/downloaders/providers.py
@@ -18,6 +18,7 @@ from six import iteritems
 import re
 from os.path import dirname, abspath, join as pathjoin
 from six.moves.urllib.parse import urlparse
+from collections import OrderedDict
 
 from .base import NoneAuthenticator, NotImplementedAuthenticator
 
@@ -199,19 +200,28 @@ class Providers(object):
         if files is None:
             # Config files from the datalad dist
             files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))
-            # User config
-            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))
 
             # Dataset config
             dsroot = get_dataset_root("")
             if dsroot is not None:
                 files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))
+
             # System config
-            files.extend(glob(pathjoin(dirs.site_config_dir, "providers", "*.cfg")))
+            if dirs.site_config_dir is not None:
+                files.extend(glob(pathjoin(dirs.site_config_dir, "providers", "*.cfg")))
+            # User config
+            if dirs.user_config_dir is not None:
+                files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))
+
+
         config.read(files)
 
         # We need first to load Providers and credentials
-        providers = {}
+        # Order matters, because we need to ensure that when
+        # there's a conflict between configuration files declared
+        # at different precedence levels (ie. dataset vs system)
+        # the appropriate precedence config wins.
+        providers = OrderedDict()
         credentials = {}
 
         for section in config.sections():
@@ -296,18 +306,21 @@ class Providers(object):
         """Given a URL returns matching provider
         """
         nproviders = len(self._providers)
-        for i in range(nproviders):
+
+        # Range backwards to ensure that more locally defined
+        # configuration wins in conflicts between url_re
+        for i in range(nproviders-1, -1, -1):
             provider = self._providers[i]
             if not provider.url_res:
                 continue
             for url_re in provider.url_res:
                 if re.match(url_re, url):
-                    if i != 0:
+                    #if i != 0:
                         # place it first
                         # TODO: optimize with smarter datastructures if this becomes a burden
-                        del self._providers[i]
-                        self._providers = [provider] + self._providers
-                        assert(len(self._providers) == nproviders)
+                        #del self._providers[i]
+                        #self._providers = [provider] + self._providers
+                        #assert(len(self._providers) == nproviders)
                     lgr.debug("Returning provider %s for url %s", provider, url)
                     return provider
 

{"datalad/downloaders/providers.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Data providers - bind downloaders and credentials together\n\"\"\"\n\nimport os\n\nfrom glob import glob\nfrom logging import getLogger\nfrom six import iteritems\n\nimport re\nfrom os.path import dirname, abspath, join as pathjoin\nfrom six.moves.urllib.parse import urlparse\n\nfrom .base import NoneAuthenticator, NotImplementedAuthenticator\n\nfrom .http import HTMLFormAuthenticator, HTTPBasicAuthAuthenticator, HTTPDigestAuthAuthenticator\nfrom .http import HTTPDownloader\nfrom .s3 import S3Authenticator, S3Downloader\nfrom ..support.configparserinc import SafeConfigParserWithIncludes\nfrom ..support.external_versions import external_versions\nfrom ..utils import assure_list_from_str\nfrom ..utils import auto_repr\nfrom ..utils import get_dataset_root\n\nfrom ..interface.common_cfg import dirs\n\nlgr = getLogger('datalad.downloaders.providers')\n\n# dict to bind authentication_type's to authenticator classes\n# parameters will be fetched from config file itself\nAUTHENTICATION_TYPES = {\n    'html_form': HTMLFormAuthenticator,\n    'http_auth': HTTPBasicAuthAuthenticator,\n    'http_basic_auth': HTTPBasicAuthAuthenticator,\n    'http_digest_auth': HTTPDigestAuthAuthenticator,\n    'aws-s3': S3Authenticator,  # TODO: check if having '-' is kosher\n    'nda-s3': S3Authenticator,\n    'xnat': NotImplementedAuthenticator,\n    'none': NoneAuthenticator,\n}\n\nfrom .credentials import CREDENTIAL_TYPES\n\ndef resolve_url_to_name(d, url):\n    \"\"\"Given a directory (e.g. of SiteInformation._items or Credential._items)\n    go through url_re and find the corresponding item and returns its key (i.e. name)\n    \"\"\"\n\n    for k, rec in iteritems(d):\n        for url_re in rec.get('url_re', '').split('\\n'):\n            if url_re:\n                if re.search(url_re, url):\n                    return k\n    return None\n\n\n@auto_repr\nclass Provider(object):\n    \"\"\"Class to bring together url_res, credential, and authenticator\n    \"\"\"\n    # TODO: we might need a lazy loading of the submodules which would provide\n    # specific downloaders while importing needed Python modules \"on demand\"\n    DOWNLOADERS = {\n        'http': {'class': HTTPDownloader, 'externals': {'requests'}},\n        'https': {'class': HTTPDownloader, 'externals': {'requests'}},\n        'ftp': {'class': HTTPDownloader, 'externals': {'requests', 'boto'}},\n        's3': {'class': S3Downloader, 'externals': {'boto'}}\n        # ... TODO\n    }\n\n    def __init__(self, name, url_res, credential=None, authenticator=None,\n                 downloader=None):\n        \"\"\"\n        Parameters\n        ----------\n        name: str\n        url_res: list of str\n           Regular expressions\n        credential: Credential, optional\n        authenticator: Authenticator, optional\n        downloader: Downloader, optional\n\n        \"\"\"\n        self.name = name\n        self.url_res = assure_list_from_str(url_res)\n        self.credential = credential\n        self.authenticator = authenticator\n        self._downloader = downloader\n\n    @property\n    def downloader(self):\n        return self._downloader\n\n    @staticmethod\n    def get_scheme_from_url(url):\n        \"\"\"Given a URL return scheme to decide which downloader class to use\n        \"\"\"\n        url_split = urlparse(url)\n        return url_split.scheme  # , url_split.netloc)\n\n    @classmethod\n    def _get_downloader_class(cls, url):\n        key = cls.get_scheme_from_url(url)\n        if key in cls.DOWNLOADERS:\n            entry = cls.DOWNLOADERS[key]\n            klass = entry['class']\n            for ext in entry.get('externals', set()):\n                if external_versions[ext] is None:\n                    raise RuntimeError(\n                        \"For using %s downloader, you need '%s' dependency \"\n                        \"which seems to be missing\" % (klass, ext)\n                    )\n            return klass\n        else:\n            raise ValueError(\"Do not know how to handle url %s for scheme %s. Known: %s\"\n                             % (url, key, cls.DOWNLOADERS.keys()))\n\n    def get_downloader(self, url, **kwargs):\n        \"\"\"Assigns proper downloader given the URL\n\n        If one is known -- verifies its appropriateness for the given url.\n        ATM we do not support multiple types of downloaders per single provider\n        \"\"\"\n        if self._downloader is None:\n            # we need to create a new one\n            Downloader = self._get_downloader_class(url)\n            # we might need to provide it with credentials and authenticator\n            # Let's do via kwargs so we could accomodate cases when downloader does not necessarily\n            # cares about those... duck typing or what it is in action\n            kwargs = kwargs.copy()\n            if self.credential:\n                kwargs['credential'] = self.credential\n            if self.authenticator:\n                kwargs['authenticator'] = self.authenticator\n            self._downloader = Downloader(**kwargs)\n        return self._downloader\n\n\nclass Providers(object):\n    \"\"\"\n\n    So we could provide handling for URLs with corresponding credentials\n    and specific (reusable) downloader.  Internally it contains\n    Providers and interfaces them based on a given URL.  Each provider\n    in turn takes care about associated with it Downloader.\n    \"\"\"\n\n    _DEFAULT_PROVIDERS = None\n\n    def __init__(self, providers=None):\n        \"\"\"\n        \"\"\"\n        self._providers = providers or []\n        # a set of providers to handle connections without authentication.\n        # Will be setup one per each protocol schema\n        self._default_providers = {}\n\n    def __repr__(self):\n        return \"%s(%s)\" % (self.__class__.__name__, \"\" if not self._providers else repr(self._providers))\n\n    def __len__(self):\n        return len(self._providers)\n\n    def __getitem__(self, index):\n        return self._providers[index]\n\n    def __iter__(self):\n        return self._providers.__iter__()\n\n    @classmethod\n    def from_config_files(cls, files=None, reload=False):\n        \"\"\"Loads information about related/possible websites requiring authentication from:\n\n        - datalad/downloaders/configs/*.cfg files provided by the codebase\n        - current dataset .datalad/providers/\n        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)\n        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)\n\n        For sample configs files see datalad/downloaders/configs/providers.cfg\n\n        If files is None, loading is cached between calls.  Specify reload=True to force\n        reloading of files from the filesystem.  The class method reset_default_providers\n        can also be called to reset the cached providers.\n        \"\"\"\n        # lazy part\n        if files is None and cls._DEFAULT_PROVIDERS and not reload:\n            return cls._DEFAULT_PROVIDERS\n\n        config = SafeConfigParserWithIncludes()\n        files_orig = files\n        if files is None:\n            # Config files from the datalad dist\n            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))\n            # User config\n            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))\n\n            # Dataset config\n            dsroot = get_dataset_root(\"\")\n            if dsroot is not None:\n                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))\n            # System config\n            files.extend(glob(pathjoin(dirs.site_config_dir, \"providers\", \"*.cfg\")))\n        config.read(files)\n\n        # We need first to load Providers and credentials\n        providers = {}\n        credentials = {}\n\n        for section in config.sections():\n            if ':' in section:\n                type_, name = section.split(':', 1)\n                assert type_ in {'provider', 'credential'}, \"we know only providers and credentials, got type %s\" % type_\n                items = {\n                    o: config.get(section, o) for o in config.options(section)\n                }\n                # side-effect -- items get poped\n                locals().get(type_ + \"s\")[name] = getattr(\n                    cls, '_process_' + type_)(name, items)\n                if len(items):\n                    raise ValueError(\"Unprocessed fields left for %s: %s\" % (name, str(items)))\n            else:\n                lgr.warning(\"Do not know how to treat section %s here\" % section)\n\n        # link credentials into providers\n        lgr.debug(\"Assigning credentials into %d providers\" % len(providers))\n        for provider in providers.values():\n            if provider.credential:\n                if provider.credential not in credentials:\n                    raise ValueError(\"Unknown credential %s. Known are: %s\"\n                                     % (provider.credential, \", \".join(credentials.keys())))\n                provider.credential = credentials[provider.credential]\n\n        providers = Providers(list(providers.values()))\n\n        if files_orig is None:\n            # Store providers for lazy access\n            cls._DEFAULT_PROVIDERS = providers\n\n        return providers\n\n    @classmethod\n    def reset_default_providers(cls):\n        \"\"\"Resets to None memoized by from_config_files providers\n        \"\"\"\n        cls._DEFAULT_PROVIDERS = None\n\n    @classmethod\n    def _process_provider(cls, name, items):\n        \"\"\"Process a dictionary specifying the provider and output the Provider instance\n        \"\"\"\n        assert 'authentication_type' in items, \"Must have authentication_type specified\"\n\n        auth_type = items.pop('authentication_type')\n        if auth_type not in AUTHENTICATION_TYPES:\n            raise ValueError(\"Unknown authentication_type=%s. Known are: %s\"\n                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))\n\n        if auth_type != 'none':\n            authenticator = AUTHENTICATION_TYPES[auth_type](\n                # Extract all the fields as keyword arguments\n                **{k[len(auth_type) + 1:]: items.pop(k)\n                   for k in list(items.keys())\n                   if k.startswith(auth_type + \"_\")}\n            )\n        else:\n            authenticator = None\n\n        # bringing url_re to \"standard\" format of a list and populating _providers_ordered\n        url_res = assure_list_from_str(items.pop('url_re'))\n        assert url_res, \"current implementation relies on having url_re defined\"\n\n        credential = items.pop('credential', None)\n\n        # credential instance will be assigned later after all of them are loaded\n        return Provider(name=name, url_res=url_res, authenticator=authenticator,\n                        credential=credential)\n\n    @classmethod\n    def _process_credential(cls, name, items):\n        assert 'type' in items, \"Credential must specify type.  Missing in %s\" % name\n        cred_type = items.pop('type')\n        if not cred_type in CREDENTIAL_TYPES:\n            raise ValueError(\"I do not know type %s credential. Known: %s\"\n                             % (cred_type, CREDENTIAL_TYPES.keys()))\n        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))\n\n    def get_provider(self, url, only_nondefault=False):\n        \"\"\"Given a URL returns matching provider\n        \"\"\"\n        nproviders = len(self._providers)\n        for i in range(nproviders):\n            provider = self._providers[i]\n            if not provider.url_res:\n                continue\n            for url_re in provider.url_res:\n                if re.match(url_re, url):\n                    if i != 0:\n                        # place it first\n                        # TODO: optimize with smarter datastructures if this becomes a burden\n                        del self._providers[i]\n                        self._providers = [provider] + self._providers\n                        assert(len(self._providers) == nproviders)\n                    lgr.debug(\"Returning provider %s for url %s\", provider, url)\n                    return provider\n\n        if only_nondefault:\n            return None\n\n        # None matched -- so we should get a default one per each of used\n        # protocols\n        scheme = Provider.get_scheme_from_url(url)\n        if scheme not in self._default_providers:\n            lgr.debug(\"Initializing default provider for %s\" % scheme)\n            self._default_providers[scheme] = Provider(name=\"\", url_res=[\"%s://.*\" % scheme])\n        provider = self._default_providers[scheme]\n        lgr.debug(\"No dedicated provider, returning default one for %s: %s\",\n                  scheme, provider)\n        return provider\n\n    # TODO: avoid duplication somehow ;)\n    # Sugarings to get easier access to downloaders\n    def download(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)\n\n    def fetch(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)\n\n    def get_status(self, url, *args, **kwargs):\n        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)\n\n    def needs_authentication(self, url):\n        provider = self.get_provider(url, only_nondefault=True)\n        if provider is None:\n            return None\n        return provider.authenticator is not None\n\n\n    # # TODO: UNUSED?\n    # def get_credentials(self, url, new=False):\n    #     \"\"\"Ask user to enter credentials for a provider matching url\n    #     \"\"\"\n    #     # find a match among _items\n    #     provider = self.get_provider(url)\n    #     if new or not provider:\n    #         rec = self._get_new_record_ui(url)\n    #         rec['url_re'] = \"TODO\"  # present to user and ask to edit\n    #         name = urlparse(url).netloc\n    #         self._items[name] = rec\n    #         if ui.yesno(\"Do you want to store credentials for %s\" % name):\n    #             self.store_credentials()\n    #     else:\n    #         return self._items[name]\n    #\n    # def store_credentials(self, name):\n    #     # TODO: store  self._items[name]  in appropriate (user) creds\n    #     # for later reuse\n    #     raise NotImplementedError()\n    #\n    # def _get_new_record_ui(self, url):\n    #     # TODO: should be a dialog with the fields appropriate for this particular\n    #     # type of credentials\n    #     ui.message(\"To access %s we would need credentials.\" % url)\n    #     if url in self.providers:\n    #         ui.message(\"If you don't yet have credentials, please visit %s\"\n    #                    % self.providers.get(url, 'credentials_url'))\n    #     return { 'user': ui.question(\"Username:\"),\n    #              'password': ui.password() }\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** TypeError When Configuration Directories Are Undefined During Provider Loading

**Description:**
When the user or site configuration directories are not set, attempting to load provider configuration files causes a `TypeError`. This error occurs because the system tries to join path components where one of the directories is `None`, leading to an unexpected failure.

**Example Code:**
```python
from datalad.downloaders.providers import Providers
from appdirs import AppDirs
from glob import glob
from os.path import join

# Simulate undefined configuration directories
AppDirs.user_config_dir = None
AppDirs.site_config_dir = None

# Attempt to load provider configurations
providers = Providers.from_config_files()
provider = providers.get_provider('https://crcns.org/data....')
```

**Expected Behavior:**
Provider configuration loading should handle `None` values for configuration directories gracefully by skipping them. The system should load available configuration files without raising errors, ensuring that provider precedence is maintained correctly.

**Actual Behavior:**
A `TypeError` is raised with the message:
```
TypeError: expected str, bytes or os.PathLike object, not NoneType
```
This error stops the provider configurations from being loaded when the configuration directories are not defined.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/downloaders/providers.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Data providers - bind downloaders and credentials together
"""

import os

from glob import glob
from logging import getLogger
from six import iteritems

import re
from os.path import dirname, abspath, join as pathjoin
from six.moves.urllib.parse import urlparse

from .base import NoneAuthenticator, NotImplementedAuthenticator

from .http import HTMLFormAuthenticator, HTTPBasicAuthAuthenticator, HTTPDigestAuthAuthenticator
from .http import HTTPDownloader
from .s3 import S3Authenticator, S3Downloader
from ..support.configparserinc import SafeConfigParserWithIncludes
from ..support.external_versions import external_versions
from ..utils import assure_list_from_str
from ..utils import auto_repr
from ..utils import get_dataset_root

from ..interface.common_cfg import dirs

lgr = getLogger('datalad.downloaders.providers')

# dict to bind authentication_type's to authenticator classes
# parameters will be fetched from config file itself
AUTHENTICATION_TYPES = {
    'html_form': HTMLFormAuthenticator,
    'http_auth': HTTPBasicAuthAuthenticator,
    'http_basic_auth': HTTPBasicAuthAuthenticator,
    'http_digest_auth': HTTPDigestAuthAuthenticator,
    'aws-s3': S3Authenticator,  # TODO: check if having '-' is kosher
    'nda-s3': S3Authenticator,
    'xnat': NotImplementedAuthenticator,
    'none': NoneAuthenticator,
}

from .credentials import CREDENTIAL_TYPES

def resolve_url_to_name(d, url):
    """Given a directory (e.g. of SiteInformation._items or Credential._items)
    go through url_re and find the corresponding item and returns its key (i.e. name)
    """

    for k, rec in iteritems(d):
        for url_re in rec.get('url_re', '').split('\n'):
            if url_re:
                if re.search(url_re, url):
                    return k
    return None


@auto_repr
class Provider(object):
    """Class to bring together url_res, credential, and authenticator
    """
    # TODO: we might need a lazy loading of the submodules which would provide
    # specific downloaders while importing needed Python modules "on demand"
    DOWNLOADERS = {
        'http': {'class': HTTPDownloader, 'externals': {'requests'}},
        'https': {'class': HTTPDownloader, 'externals': {'requests'}},
        'ftp': {'class': HTTPDownloader, 'externals': {'requests', 'boto'}},
        's3': {'class': S3Downloader, 'externals': {'boto'}}
        # ... TODO
    }

    def __init__(self, name, url_res, credential=None, authenticator=None,
                 downloader=None):
        """
        Parameters
        ----------
        name: str
        url_res: list of str
           Regular expressions
        credential: Credential, optional
        authenticator: Authenticator, optional
        downloader: Downloader, optional

        """
        self.name = name
        self.url_res = assure_list_from_str(url_res)
        self.credential = credential
        self.authenticator = authenticator
        self._downloader = downloader

    @property
    def downloader(self):
        return self._downloader

    @staticmethod
    def get_scheme_from_url(url):
        """Given a URL return scheme to decide which downloader class to use
        """
        url_split = urlparse(url)
        return url_split.scheme  # , url_split.netloc)

    @classmethod
    def _get_downloader_class(cls, url):
        key = cls.get_scheme_from_url(url)
        if key in cls.DOWNLOADERS:
            entry = cls.DOWNLOADERS[key]
            klass = entry['class']
            for ext in entry.get('externals', set()):
                if external_versions[ext] is None:
                    raise RuntimeError(
                        "For using %s downloader, you need '%s' dependency "
                        "which seems to be missing" % (klass, ext)
                    )
            return klass
        else:
            raise ValueError("Do not know how to handle url %s for scheme %s. Known: %s"
                             % (url, key, cls.DOWNLOADERS.keys()))

    def get_downloader(self, url, **kwargs):
        """Assigns proper downloader given the URL

        If one is known -- verifies its appropriateness for the given url.
        ATM we do not support multiple types of downloaders per single provider
        """
        if self._downloader is None:
            # we need to create a new one
            Downloader = self._get_downloader_class(url)
            # we might need to provide it with credentials and authenticator
            # Let's do via kwargs so we could accomodate cases when downloader does not necessarily
            # cares about those... duck typing or what it is in action
            kwargs = kwargs.copy()
            if self.credential:
                kwargs['credential'] = self.credential
            if self.authenticator:
                kwargs['authenticator'] = self.authenticator
            self._downloader = Downloader(**kwargs)
        return self._downloader


class Providers(object):
    """

    So we could provide handling for URLs with corresponding credentials
    and specific (reusable) downloader.  Internally it contains
    Providers and interfaces them based on a given URL.  Each provider
    in turn takes care about associated with it Downloader.
    """

    _DEFAULT_PROVIDERS = None

    def __init__(self, providers=None):
        """
        """
        self._providers = providers or []
        # a set of providers to handle connections without authentication.
        # Will be setup one per each protocol schema
        self._default_providers = {}

    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, "" if not self._providers else repr(self._providers))

    def __len__(self):
        return len(self._providers)

    def __getitem__(self, index):
        return self._providers[index]

    def __iter__(self):
        return self._providers.__iter__()

    @classmethod
    def from_config_files(cls, files=None, reload=False):
        """Loads information about related/possible websites requiring authentication from:

        - datalad/downloaders/configs/*.cfg files provided by the codebase
        - current dataset .datalad/providers/
        - User's home directory directory (ie ~/.config/datalad/providers/*.cfg)
        - system-wide datalad installation/config (ie /etc/datalad/providers/*.cfg)

        For sample configs files see datalad/downloaders/configs/providers.cfg

        If files is None, loading is cached between calls.  Specify reload=True to force
        reloading of files from the filesystem.  The class method reset_default_providers
        can also be called to reset the cached providers.
        """
        # lazy part
        if files is None and cls._DEFAULT_PROVIDERS and not reload:
            return cls._DEFAULT_PROVIDERS

        config = SafeConfigParserWithIncludes()
        files_orig = files
        if files is None:
            # Config files from the datalad dist
            files = glob(pathjoin(dirname(abspath(__file__)), 'configs', '*.cfg'))
            # User config
            files.extend(glob(pathjoin(dirs.user_config_dir, 'providers', '*.cfg')))

            # Dataset config
            dsroot = get_dataset_root("")
            if dsroot is not None:
                files.extend(glob(pathjoin(dsroot, '.datalad', 'providers', '*.cfg')))
            # System config
            files.extend(glob(pathjoin(dirs.site_config_dir, "providers", "*.cfg")))
        config.read(files)

        # We need first to load Providers and credentials
        providers = {}
        credentials = {}

        for section in config.sections():
            if ':' in section:
                type_, name = section.split(':', 1)
                assert type_ in {'provider', 'credential'}, "we know only providers and credentials, got type %s" % type_
                items = {
                    o: config.get(section, o) for o in config.options(section)
                }
                # side-effect -- items get poped
                locals().get(type_ + "s")[name] = getattr(
                    cls, '_process_' + type_)(name, items)
                if len(items):
                    raise ValueError("Unprocessed fields left for %s: %s" % (name, str(items)))
            else:
                lgr.warning("Do not know how to treat section %s here" % section)

        # link credentials into providers
        lgr.debug("Assigning credentials into %d providers" % len(providers))
        for provider in providers.values():
            if provider.credential:
                if provider.credential not in credentials:
                    raise ValueError("Unknown credential %s. Known are: %s"
                                     % (provider.credential, ", ".join(credentials.keys())))
                provider.credential = credentials[provider.credential]

        providers = Providers(list(providers.values()))

        if files_orig is None:
            # Store providers for lazy access
            cls._DEFAULT_PROVIDERS = providers

        return providers

    @classmethod
    def reset_default_providers(cls):
        """Resets to None memoized by from_config_files providers
        """
        cls._DEFAULT_PROVIDERS = None

    @classmethod
    def _process_provider(cls, name, items):
        """Process a dictionary specifying the provider and output the Provider instance
        """
        assert 'authentication_type' in items, "Must have authentication_type specified"

        auth_type = items.pop('authentication_type')
        if auth_type not in AUTHENTICATION_TYPES:
            raise ValueError("Unknown authentication_type=%s. Known are: %s"
                             % (auth_type, ', '.join(AUTHENTICATION_TYPES)))

        if auth_type != 'none':
            authenticator = AUTHENTICATION_TYPES[auth_type](
                # Extract all the fields as keyword arguments
                **{k[len(auth_type) + 1:]: items.pop(k)
                   for k in list(items.keys())
                   if k.startswith(auth_type + "_")}
            )
        else:
            authenticator = None

        # bringing url_re to "standard" format of a list and populating _providers_ordered
        url_res = assure_list_from_str(items.pop('url_re'))
        assert url_res, "current implementation relies on having url_re defined"

        credential = items.pop('credential', None)

        # credential instance will be assigned later after all of them are loaded
        return Provider(name=name, url_res=url_res, authenticator=authenticator,
                        credential=credential)

    @classmethod
    def _process_credential(cls, name, items):
        assert 'type' in items, "Credential must specify type.  Missing in %s" % name
        cred_type = items.pop('type')
        if not cred_type in CREDENTIAL_TYPES:
            raise ValueError("I do not know type %s credential. Known: %s"
                             % (cred_type, CREDENTIAL_TYPES.keys()))
        return CREDENTIAL_TYPES[cred_type](name=name, url=items.pop('url', None))

    def get_provider(self, url, only_nondefault=False):
        """Given a URL returns matching provider
        """
        nproviders = len(self._providers)
        for i in range(nproviders):
            provider = self._providers[i]
            if not provider.url_res:
                continue
            for url_re in provider.url_res:
                if re.match(url_re, url):
                    if i != 0:
                        # place it first
                        # TODO: optimize with smarter datastructures if this becomes a burden
                        del self._providers[i]
                        self._providers = [provider] + self._providers
                        assert(len(self._providers) == nproviders)
                    lgr.debug("Returning provider %s for url %s", provider, url)
                    return provider

        if only_nondefault:
            return None

        # None matched -- so we should get a default one per each of used
        # protocols
        scheme = Provider.get_scheme_from_url(url)
        if scheme not in self._default_providers:
            lgr.debug("Initializing default provider for %s" % scheme)
            self._default_providers[scheme] = Provider(name="", url_res=["%s://.*" % scheme])
        provider = self._default_providers[scheme]
        lgr.debug("No dedicated provider, returning default one for %s: %s",
                  scheme, provider)
        return provider

    # TODO: avoid duplication somehow ;)
    # Sugarings to get easier access to downloaders
    def download(self, url, *args, **kwargs):
        return self.get_provider(url).get_downloader(url).download(url, *args, **kwargs)

    def fetch(self, url, *args, **kwargs):
        return self.get_provider(url).get_downloader(url).fetch(url, *args, **kwargs)

    def get_status(self, url, *args, **kwargs):
        return self.get_provider(url).get_downloader(url).get_status(url, *args, **kwargs)

    def needs_authentication(self, url):
        provider = self.get_provider(url, only_nondefault=True)
        if provider is None:
            return None
        return provider.authenticator is not None


    # # TODO: UNUSED?
    # def get_credentials(self, url, new=False):
    #     """Ask user to enter credentials for a provider matching url
    #     """
    #     # find a match among _items
    #     provider = self.get_provider(url)
    #     if new or not provider:
    #         rec = self._get_new_record_ui(url)
    #         rec['url_re'] = "TODO"  # present to user and ask to edit
    #         name = urlparse(url).netloc
    #         self._items[name] = rec
    #         if ui.yesno("Do you want to store credentials for %s" % name):
    #             self.store_credentials()
    #     else:
    #         return self._items[name]
    #
    # def store_credentials(self, name):
    #     # TODO: store  self._items[name]  in appropriate (user) creds
    #     # for later reuse
    #     raise NotImplementedError()
    #
    # def _get_new_record_ui(self, url):
    #     # TODO: should be a dialog with the fields appropriate for this particular
    #     # type of credentials
    #     ui.message("To access %s we would need credentials." % url)
    #     if url in self.providers:
    #         ui.message("If you don't yet have credentials, please visit %s"
    #                    % self.providers.get(url, 'credentials_url'))
    #     return { 'user': ui.question("Username:"),
    #              'password': ui.password() }

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user