{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface information about credentials\n\nProvides minimalistic interface to deal (query, request, store) with most common\ntypes of credentials.  To be used by Authenticators\n\"\"\"\n\n__dev_doc__ = \"\"\"\nPossibly useful in the future 3rd part developments\n\nhttps://github.com/omab/python-social-auth\n    social authentication/registration mechanism with support for several\n    frameworks and auth providers\n\"\"\"\n\nimport time\nimport calendar\n\nfrom collections import OrderedDict\n\nfrom ..dochelpers import exc_str\nfrom ..support.keyring_ import keyring as keyring_\nfrom ..ui import ui\nfrom ..utils import auto_repr\nfrom ..support.network import iso8601_to_epoch\n\nfrom logging import getLogger\nlgr = getLogger('datalad.downloaders.credentials')\n\n\n@auto_repr\nclass Credential(object):\n    \"\"\"Base class for different types of credentials\n    \"\"\"\n\n    # Should be defined in a subclass as an OrderedDict of fields\n    # name -> dict(attributes)\n    _FIELDS = None\n    _KNOWN_ATTRS = {\n        'hidden',    # UI should not display the value\n        'optional',  # Not mandatory thus not requested if not set\n    }\n\n    def __init__(self, name, url=None, keyring=None):\n        \"\"\"\n        Parameters\n        ----------\n        name : str\n            Name of the credential, as it would be identified by in the centralized\n            storage of credentials\n        url : str, optional\n            URL string to point users to where to seek obtaining the credentials\n        keyring : a keyring\n            An object providing (g|s)et_password.  If None, keyring module is used\n            as is\n        \"\"\"\n        self.name = name\n        self.url = url\n        self._keyring = keyring or keyring_\n        self._prepare()\n\n    def _prepare(self):\n        \"\"\"Additional house-keeping possibly to be performed in subclasses\n\n        Created to avoid all the passing args/kwargs of __init__ all the time\n        \"\"\"\n        # Basic checks\n        for f, fattrs in self._FIELDS.items():\n            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)\n            if unknown_attrs:\n                raise ValueError(\"Unknown attributes %s. Known are: %s\"\n                                 % (unknown_attrs, self._KNOWN_ATTRS))\n\n    def _is_field_optional(self, f):\n        return self._FIELDS[f].get('optional', False)\n\n    def _is_field_hidden(self, f):\n        return self._FIELDS[f].get('hidden', False)\n\n    @property\n    def is_known(self):\n        \"\"\"Return True if values for all fields of the credential are known\"\"\"\n        try:\n            return all(\n                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None\n                for f in self._FIELDS)\n        except Exception as exc:\n            lgr.warning(\"Failed to query keyring: %s\" % exc_str(exc))\n            return False\n\n    def _ask_field_value(self, f, instructions=None):\n        msg = instructions if instructions else \\\n            (\"You need to authenticate with %r credentials.\" % self.name +\n                  (\" %s provides information on how to gain access\"\n                   % self.url if self.url else ''))\n\n        return ui.question(\n            f,\n            title=msg,\n            hidden=self._is_field_hidden(f))\n\n    def _ask_and_set(self, f, instructions=None):\n        v = self._ask_field_value(f, instructions=instructions)\n        self.set(**{f: v})\n        return v\n\n    def enter_new(self, instructions=None, **kwargs):\n        \"\"\"Enter new values for the credential fields\n\n        Parameters\n        ----------\n        instructions : str, optional\n          If given, the auto-generated instructions based on a login-URL are\n          replaced by the given string\n        **kwargs\n          Any given key value pairs with non-None values are used to set the\n          field `key` to the given value, without asking for user input\n        \"\"\"\n        # Use ui., request credential fields corresponding to the type\n        for f in self._FIELDS:\n            if kwargs.get(f, None):\n                # use given value, don't ask\n                self.set(**{f: kwargs[f]})\n            elif not self._is_field_optional(f):\n                self._ask_and_set(f, instructions=instructions)\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        name = self.name\n        fields = {}\n        for f in self._FIELDS:\n            v = self._keyring.get(name, f)\n            if not self._is_field_optional(f):\n                while v is None:  # was not known\n                    v = self._ask_and_set(f)\n                fields[f] = v\n            elif v is not None:\n                fields[f] = v\n        return fields\n\n    def set(self, **kwargs):\n        \"\"\"Set field(s) of the credential\"\"\"\n        for f, v in kwargs.items():\n            if f not in self._FIELDS:\n                raise ValueError(\"Unknown field %s. Known are: %s\"\n                                 % (f, self._FIELDS.keys()))\n            self._keyring.set(self.name, f, v)\n\n    def get(self, f, default=None):\n        \"\"\"Get a field of the credential\"\"\"\n        if f not in self._FIELDS:\n            raise ValueError(\"Unknown field %s. Known are: %s\"\n                             % (f, self._FIELDS.keys()))\n        try:\n            return self._keyring.get(self.name, f)\n        except:  # MIH: what could even happen? _keyring not a dict?\n            return default\n\n    def delete(self):\n        \"\"\"Deletes credential values from the keyring\"\"\"\n        for f in self._FIELDS:\n            self._keyring.delete(self.name, f)\n\n\nclass UserPassword(Credential):\n    \"\"\"Simple type of a credential which consists of user/password pair\"\"\"\n\n    _FIELDS = OrderedDict([('user', {}),\n                           ('password', {'hidden': True})])\n\n    is_expired = False  # no expiration provisioned\n\n\nclass Token(Credential):\n    \"\"\"Simple type of a credential which provides a single token\"\"\"\n\n    _FIELDS = OrderedDict([('token', {'hidden': True})])\n\n    is_expired = False  # no expiration provisioned\n\n\nclass AWS_S3(Credential):\n    \"\"\"Credential for AWS S3 service\"\"\"\n\n    _FIELDS = OrderedDict([('key_id', {}),\n                           ('secret_id', {'hidden': True}),\n                           ('session', {'optional': True}),\n                           ('expiration', {'optional': True}),\n                           ])\n\n    @property\n    def is_expired(self):\n        exp = self.get('expiration', None)\n        if not exp:\n            return True\n        exp_epoch = iso8601_to_epoch(exp)\n        expire_in = (exp_epoch - calendar.timegm(time.localtime())) / 3600.\n\n        lgr.debug(\n            (\"Credential %s has expired %.2fh ago\"\n                if expire_in <= 0 else \"Credential %s will expire in %.2fh\")\n            % (self.name, expire_in))\n        return expire_in <= 0\n\n\n@auto_repr\nclass CompositeCredential(Credential):\n    \"\"\"Credential which represent a sequence of Credentials where front one is exposed to user\n    \"\"\"\n\n    # To be defined in sub-classes\n    _CREDENTIAL_CLASSES = None\n    _CREDENTIAL_ADAPTERS = None\n\n    def _prepare(self):\n        assert len(self._CREDENTIAL_CLASSES) > 1, \"makes sense only if there is > 1 credential\"\n        assert len(self._CREDENTIAL_CLASSES) == len(self._CREDENTIAL_ADAPTERS) + 1, \\\n            \"there should be 1 less of adapter than _CREDENTIAL_CLASSES\"\n\n        for C in self._CREDENTIAL_CLASSES:\n            assert issubclass(C, Credential), \"%s must be a subclass of Credential\" % C\n\n        # First credential should bear the name and url\n        credentials = [self._CREDENTIAL_CLASSES[0](self.name, url=self.url, keyring=self._keyring)]\n        # and we just reuse its _FIELDS for _ask_field_value etc\n        self._FIELDS = credentials[0]._FIELDS\n        # the rest with index suffix, but storing themselves in the same keyring\n        for iC, C in enumerate(self._CREDENTIAL_CLASSES[1:]):\n            credentials.append(\n                C(name=\"%s:%d\" % (self.name, iC + 1), url=None, keyring=self._keyring)\n            )\n        self._credentials = credentials\n\n        super(CompositeCredential, self)._prepare()\n\n    # Here it becomes tricky, since theoretically it is the \"tail\"\n    # ones which might expire etc, so we wouldn't exactly know what\n    # new credentials outside process wanted -- it would be silly to ask\n    # right away the \"entry\" credentials if it is just the expiration of the\n    # tail credentials\n    def enter_new(self):\n        # should invalidate/remove all tail credentials to avoid failing attempts to login\n        self._credentials[0].enter_new()\n        for c in self._credentials[1:]:\n            c.delete()\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        # Start from the tail until we have credentials set\n        idx = len(self._credentials) - 1\n        for c in self._credentials[::-1]:\n            if c.is_known and not c.is_expired:\n                break\n            idx -= 1\n\n        if idx < 0:\n            # none was known, all the same -- start with the first one\n            idx = 0\n\n        # TODO: consider moving logic of traversal into authenticator since it is\n        # the one spitting out authentication error etc\n        # Theoretically we could just reuse 'fields' from adapter in the next step\n        # but let's do full circle, so that if any \"normalization\" is done by\n        # Credentials we take that into account\n        for c, adapter, next_c in zip(\n                self._credentials[idx:],\n                self._CREDENTIAL_ADAPTERS[idx:],\n                self._credentials[idx + 1:]):\n            fields = c()\n            next_fields = adapter(self, **fields)\n            next_c.set(**next_fields)\n\n        return self._credentials[-1]()\n\n\ndef _nda_adapter(composite, user=None, password=None):\n    from datalad.support.third.nda_aws_token_generator import NDATokenGenerator\n    gen = NDATokenGenerator()\n    token = gen.generate_token(user, password)\n    # There are also session and expiration we ignore... TODO anything about it?!!!\n    # we could create a derived AWS_S3 which would also store session and expiration\n    # and then may be Composite could use those????\n    return dict(key_id=token.access_key, secret_id=token.secret_key,\n                session=token.session, expiration=token.expiration)\n\n\nclass NDA_S3(CompositeCredential):\n    \"\"\"Credential to access NDA AWS\n\n    So for NDA we need a credential which is a composite credential.\n    User provides UserPassword and then some adapter generates AWS_S3\n    out of it\n    \"\"\"\n    _CREDENTIAL_CLASSES = (UserPassword, AWS_S3)\n    _CREDENTIAL_ADAPTERS = (_nda_adapter,)\n\n\ndef _loris_adapter(composite, user=None, password=None, **kwargs):\n    from datalad.support.third.loris_token_generator import LORISTokenGenerator\n\n    gen = LORISTokenGenerator(url=composite.url)\n    token = gen.generate_token(user, password)\n\n    return dict(token=token)\n\n\nclass LORIS_Token(CompositeCredential):\n    _CREDENTIAL_CLASSES = (UserPassword, Token)\n    _CREDENTIAL_ADAPTERS = (_loris_adapter,)\n\n    def __init__(self, name, url=None, keyring=None):\n        super(CompositeCredential, self).__init__(name, url, keyring)\n\n\nCREDENTIAL_TYPES = {\n    'user_password': UserPassword,\n    'aws-s3': AWS_S3,\n    'nda-s3': NDA_S3,\n    'token': Token,\n    'loris-token': LORIS_Token\n}\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface information about credentials\n\nProvides minimalistic interface to deal (query, request, store) with most common\ntypes of credentials.  To be used by Authenticators\n\"\"\"\n\n__dev_doc__ = \"\"\"\nPossibly useful in the future 3rd part developments\n\nhttps://github.com/omab/python-social-auth\n    social authentication/registration mechanism with support for several\n    frameworks and auth providers\n\"\"\"\n\nimport time\nimport calendar\n\nfrom collections import OrderedDict\n\nfrom ..dochelpers import exc_str\nfrom ..support.keyring_ import keyring as keyring_\nfrom ..ui import ui\nfrom ..utils import auto_repr\nfrom ..support.network import iso8601_to_epoch\n\nfrom logging import getLogger\nlgr = getLogger('datalad.downloaders.credentials')\n\n\n@auto_repr\nclass Credential(object):\n    \"\"\"Base class for different types of credentials\n    \"\"\"\n\n    # Should be defined in a subclass as an OrderedDict of fields\n    # name -> dict(attributes)\n    _FIELDS = None\n    _KNOWN_ATTRS = {\n        'hidden',    # UI should not display the value\n        'optional',  # Not mandatory thus not requested if not set\n    }\n\n    def __init__(self, name, url=None, keyring=None):\n        \"\"\"\n        Parameters\n        ----------\n        name : str\n            Name of the credential, as it would be identified by in the centralized\n            storage of credentials\n        url : str, optional\n            URL string to point users to where to seek obtaining the credentials\n        keyring : a keyring\n            An object providing (g|s)et_password.  If None, keyring module is used\n            as is\n        \"\"\"\n        self.name = name\n        self.url = url\n        self._keyring = keyring or keyring_\n        self._prepare()\n\n    def _prepare(self):\n        \"\"\"Additional house-keeping possibly to be performed in subclasses\n\n        Created to avoid all the passing args/kwargs of __init__ all the time\n        \"\"\"\n        # Basic checks\n        for f, fattrs in self._FIELDS.items():\n            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)\n            if unknown_attrs:\n                raise ValueError(\"Unknown attributes %s. Known are: %s\"\n                                 % (unknown_attrs, self._KNOWN_ATTRS))\n\n    def _is_field_optional(self, f):\n        return self._FIELDS[f].get('optional', False)\n\n    def _is_field_hidden(self, f):\n        return self._FIELDS[f].get('hidden', False)\n\n    @property\n    def is_known(self):\n        \"\"\"Return True if values for all fields of the credential are known\"\"\"\n        try:\n            return all(\n                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None\n                for f in self._FIELDS)\n        except Exception as exc:\n            lgr.warning(\"Failed to query keyring: %s\" % exc_str(exc))\n            return False\n\n    def _ask_field_value(self, f, instructions=None):\n        msg = instructions if instructions else \\\n            (\"You need to authenticate with %r credentials.\" % self.name +\n                  (\" %s provides information on how to gain access\"\n                   % self.url if self.url else ''))\n\n        return ui.question(\n            f,\n            title=msg,\n            hidden=self._is_field_hidden(f))\n\n    def _ask_and_set(self, f, instructions=None):\n        v = self._ask_field_value(f, instructions=instructions)\n        self.set(**{f: v})\n        return v\n\n    def enter_new(self, instructions=None, **kwargs):\n        \"\"\"Enter new values for the credential fields\n\n        Parameters\n        ----------\n        instructions : str, optional\n          If given, the auto-generated instructions based on a login-URL are\n          replaced by the given string\n        **kwargs\n          Any given key value pairs with non-None values are used to set the\n          field `key` to the given value, without asking for user input\n        \"\"\"\n        if kwargs:\n            unknown_fields = set(kwargs).difference(self._FIELDS)\n            known_fields = set(self._FIELDS).difference(kwargs)\n            if unknown_fields:\n                raise ValueError(\n                    \"Unknown to %s field(s): %s.  Known but not specified: %s\"\n                    % (self,\n                       ', '.join(sorted(unknown_fields)),\n                       ', '.join(sorted(known_fields))\n                       ))\n        # Use ui., request credential fields corresponding to the type\n        for f in self._FIELDS:\n            if kwargs.get(f, None):\n                # use given value, don't ask\n                self.set(**{f: kwargs[f]})\n            elif not self._is_field_optional(f):\n                self._ask_and_set(f, instructions=instructions)\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        name = self.name\n        fields = {}\n        for f in self._FIELDS:\n            v = self._keyring.get(name, f)\n            if not self._is_field_optional(f):\n                while v is None:  # was not known\n                    v = self._ask_and_set(f)\n                fields[f] = v\n            elif v is not None:\n                fields[f] = v\n        return fields\n\n    def set(self, **kwargs):\n        \"\"\"Set field(s) of the credential\"\"\"\n        for f, v in kwargs.items():\n            if f not in self._FIELDS:\n                raise ValueError(\"Unknown field %s. Known are: %s\"\n                                 % (f, self._FIELDS.keys()))\n            self._keyring.set(self.name, f, v)\n\n    def get(self, f, default=None):\n        \"\"\"Get a field of the credential\"\"\"\n        if f not in self._FIELDS:\n            raise ValueError(\"Unknown field %s. Known are: %s\"\n                             % (f, self._FIELDS.keys()))\n        try:\n            return self._keyring.get(self.name, f)\n        except:  # MIH: what could even happen? _keyring not a dict?\n            return default\n\n    def delete(self):\n        \"\"\"Deletes credential values from the keyring\"\"\"\n        for f in self._FIELDS:\n            self._keyring.delete(self.name, f)\n\n\nclass UserPassword(Credential):\n    \"\"\"Simple type of a credential which consists of user/password pair\"\"\"\n\n    _FIELDS = OrderedDict([('user', {}),\n                           ('password', {'hidden': True})])\n\n    is_expired = False  # no expiration provisioned\n\n\nclass Token(Credential):\n    \"\"\"Simple type of a credential which provides a single token\"\"\"\n\n    _FIELDS = OrderedDict([('token', {'hidden': True})])\n\n    is_expired = False  # no expiration provisioned\n\n\nclass AWS_S3(Credential):\n    \"\"\"Credential for AWS S3 service\"\"\"\n\n    _FIELDS = OrderedDict([('key_id', {}),\n                           ('secret_id', {'hidden': True}),\n                           ('session', {'optional': True}),\n                           ('expiration', {'optional': True}),\n                           ])\n\n    @property\n    def is_expired(self):\n        exp = self.get('expiration', None)\n        if not exp:\n            return True\n        exp_epoch = iso8601_to_epoch(exp)\n        expire_in = (exp_epoch - calendar.timegm(time.localtime())) / 3600.\n\n        lgr.debug(\n            (\"Credential %s has expired %.2fh ago\"\n                if expire_in <= 0 else \"Credential %s will expire in %.2fh\")\n            % (self.name, expire_in))\n        return expire_in <= 0\n\n\n@auto_repr\nclass CompositeCredential(Credential):\n    \"\"\"Credential which represent a sequence of Credentials where front one is exposed to user\n    \"\"\"\n\n    # To be defined in sub-classes\n    _CREDENTIAL_CLASSES = None\n    _CREDENTIAL_ADAPTERS = None\n\n    def _prepare(self):\n        assert len(self._CREDENTIAL_CLASSES) > 1, \"makes sense only if there is > 1 credential\"\n        assert len(self._CREDENTIAL_CLASSES) == len(self._CREDENTIAL_ADAPTERS) + 1, \\\n            \"there should be 1 less of adapter than _CREDENTIAL_CLASSES\"\n\n        for C in self._CREDENTIAL_CLASSES:\n            assert issubclass(C, Credential), \"%s must be a subclass of Credential\" % C\n\n        # First credential should bear the name and url\n        credentials = [self._CREDENTIAL_CLASSES[0](self.name, url=self.url, keyring=self._keyring)]\n        # and we just reuse its _FIELDS for _ask_field_value etc\n        self._FIELDS = credentials[0]._FIELDS\n        # the rest with index suffix, but storing themselves in the same keyring\n        for iC, C in enumerate(self._CREDENTIAL_CLASSES[1:]):\n            credentials.append(\n                C(name=\"%s:%d\" % (self.name, iC + 1), url=None, keyring=self._keyring)\n            )\n        self._credentials = credentials\n\n        super(CompositeCredential, self)._prepare()\n\n    # Here it becomes tricky, since theoretically it is the \"tail\"\n    # ones which might expire etc, so we wouldn't exactly know what\n    # new credentials outside process wanted -- it would be silly to ask\n    # right away the \"entry\" credentials if it is just the expiration of the\n    # tail credentials\n    def enter_new(self):\n        # should invalidate/remove all tail credentials to avoid failing attempts to login\n        self._credentials[0].enter_new()\n        for c in self._credentials[1:]:\n            c.delete()\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        # Start from the tail until we have credentials set\n        idx = len(self._credentials) - 1\n        for c in self._credentials[::-1]:\n            if c.is_known and not c.is_expired:\n                break\n            idx -= 1\n\n        if idx < 0:\n            # none was known, all the same -- start with the first one\n            idx = 0\n\n        # TODO: consider moving logic of traversal into authenticator since it is\n        # the one spitting out authentication error etc\n        # Theoretically we could just reuse 'fields' from adapter in the next step\n        # but let's do full circle, so that if any \"normalization\" is done by\n        # Credentials we take that into account\n        for c, adapter, next_c in zip(\n                self._credentials[idx:],\n                self._CREDENTIAL_ADAPTERS[idx:],\n                self._credentials[idx + 1:]):\n            fields = c()\n            next_fields = adapter(self, **fields)\n            next_c.set(**next_fields)\n\n        return self._credentials[-1]()\n\n\ndef _nda_adapter(composite, user=None, password=None):\n    from datalad.support.third.nda_aws_token_generator import NDATokenGenerator\n    gen = NDATokenGenerator()\n    token = gen.generate_token(user, password)\n    # There are also session and expiration we ignore... TODO anything about it?!!!\n    # we could create a derived AWS_S3 which would also store session and expiration\n    # and then may be Composite could use those????\n    return dict(key_id=token.access_key, secret_id=token.secret_key,\n                session=token.session, expiration=token.expiration)\n\n\nclass NDA_S3(CompositeCredential):\n    \"\"\"Credential to access NDA AWS\n\n    So for NDA we need a credential which is a composite credential.\n    User provides UserPassword and then some adapter generates AWS_S3\n    out of it\n    \"\"\"\n    _CREDENTIAL_CLASSES = (UserPassword, AWS_S3)\n    _CREDENTIAL_ADAPTERS = (_nda_adapter,)\n\n\ndef _loris_adapter(composite, user=None, password=None, **kwargs):\n    from datalad.support.third.loris_token_generator import LORISTokenGenerator\n\n    gen = LORISTokenGenerator(url=composite.url)\n    token = gen.generate_token(user, password)\n\n    return dict(token=token)\n\n\nclass LORIS_Token(CompositeCredential):\n    _CREDENTIAL_CLASSES = (UserPassword, Token)\n    _CREDENTIAL_ADAPTERS = (_loris_adapter,)\n\n    def __init__(self, name, url=None, keyring=None):\n        super(CompositeCredential, self).__init__(name, url, keyring)\n\n\nCREDENTIAL_TYPES = {\n    'user_password': UserPassword,\n    'aws-s3': AWS_S3,\n    'nda-s3': NDA_S3,\n    'token': Token,\n    'loris-token': LORIS_Token\n}\n",
            "header": {
                "file": {
                    "path": "datalad/downloaders/credentials.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "a486c9c92",
                "new_commit_hash": "22446eed1",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/downloaders/credentials.py"
            },
            "plus_file": {
                "path": "b/datalad/downloaders/credentials.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 123,
                            "length": 6
                        },
                        "new_range": {
                            "start": 123,
                            "length": 16
                        },
                        "section": "class Credential(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "          Any given key value pairs with non-None values are used to set the",
                                "type": "context"
                            },
                            {
                                "content": "          field `key` to the given value, without asking for user input",
                                "type": "context"
                            },
                            {
                                "content": "        \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "        if kwargs:",
                                "type": "added"
                            },
                            {
                                "content": "            unknown_fields = set(kwargs).difference(self._FIELDS)",
                                "type": "added"
                            },
                            {
                                "content": "            known_fields = set(self._FIELDS).difference(kwargs)",
                                "type": "added"
                            },
                            {
                                "content": "            if unknown_fields:",
                                "type": "added"
                            },
                            {
                                "content": "                raise ValueError(",
                                "type": "added"
                            },
                            {
                                "content": "                    \"Unknown to %s field(s): %s.  Known but not specified: %s\"",
                                "type": "added"
                            },
                            {
                                "content": "                    % (self,",
                                "type": "added"
                            },
                            {
                                "content": "                       ', '.join(sorted(unknown_fields)),",
                                "type": "added"
                            },
                            {
                                "content": "                       ', '.join(sorted(known_fields))",
                                "type": "added"
                            },
                            {
                                "content": "                       ))",
                                "type": "added"
                            },
                            {
                                "content": "        # Use ui., request credential fields corresponding to the type",
                                "type": "context"
                            },
                            {
                                "content": "        for f in self._FIELDS:",
                                "type": "context"
                            },
                            {
                                "content": "            if kwargs.get(f, None):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/downloaders/credentials.py",
                            "type": "class",
                            "name": "Credential",
                            "content": "class Credential(object):\n    \"\"\"Base class for different types of credentials\n    \"\"\"\n\n    # Should be defined in a subclass as an OrderedDict of fields\n    # name -> dict(attributes)\n    _FIELDS = None\n    _KNOWN_ATTRS = {\n        'hidden',    # UI should not display the value\n        'optional',  # Not mandatory thus not requested if not set\n    }\n\n    def __init__(self, name, url=None, keyring=None):\n        \"\"\"\n        Parameters\n        ----------\n        name : str\n            Name of the credential, as it would be identified by in the centralized\n            storage of credentials\n        url : str, optional\n            URL string to point users to where to seek obtaining the credentials\n        keyring : a keyring\n            An object providing (g|s)et_password.  If None, keyring module is used\n            as is\n        \"\"\"\n        self.name = name\n        self.url = url\n        self._keyring = keyring or keyring_\n        self._prepare()\n\n    def _prepare(self):\n        \"\"\"Additional house-keeping possibly to be performed in subclasses\n\n        Created to avoid all the passing args/kwargs of __init__ all the time\n        \"\"\"\n        # Basic checks\n        for f, fattrs in self._FIELDS.items():\n            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)\n            if unknown_attrs:\n                raise ValueError(\"Unknown attributes %s. Known are: %s\"\n                                 % (unknown_attrs, self._KNOWN_ATTRS))\n\n    def _is_field_optional(self, f):\n        return self._FIELDS[f].get('optional', False)\n\n    def _is_field_hidden(self, f):\n        return self._FIELDS[f].get('hidden', False)\n\n    @property\n    def is_known(self):\n        \"\"\"Return True if values for all fields of the credential are known\"\"\"\n        try:\n            return all(\n                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None\n                for f in self._FIELDS)\n        except Exception as exc:\n            lgr.warning(\"Failed to query keyring: %s\" % exc_str(exc))\n            return False\n\n    def _ask_field_value(self, f, instructions=None):\n        msg = instructions if instructions else \\\n            (\"You need to authenticate with %r credentials.\" % self.name +\n                  (\" %s provides information on how to gain access\"\n                   % self.url if self.url else ''))\n\n        return ui.question(\n            f,\n            title=msg,\n            hidden=self._is_field_hidden(f))\n\n    def _ask_and_set(self, f, instructions=None):\n        v = self._ask_field_value(f, instructions=instructions)\n        self.set(**{f: v})\n        return v\n\n    def enter_new(self, instructions=None, **kwargs):\n        \"\"\"Enter new values for the credential fields\n\n        Parameters\n        ----------\n        instructions : str, optional\n          If given, the auto-generated instructions based on a login-URL are\n          replaced by the given string\n        **kwargs\n          Any given key value pairs with non-None values are used to set the\n          field `key` to the given value, without asking for user input\n        \"\"\"\n        if kwargs:\n            unknown_fields = set(kwargs).difference(self._FIELDS)\n            known_fields = set(self._FIELDS).difference(kwargs)\n            if unknown_fields:\n                raise ValueError(\n                    \"Unknown to %s field(s): %s.  Known but not specified: %s\"\n                    % (self,\n                       ', '.join(sorted(unknown_fields)),\n                       ', '.join(sorted(known_fields))\n                       ))\n        # Use ui., request credential fields corresponding to the type\n        for f in self._FIELDS:\n            if kwargs.get(f, None):\n                # use given value, don't ask\n                self.set(**{f: kwargs[f]})\n            elif not self._is_field_optional(f):\n                self._ask_and_set(f, instructions=instructions)\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        name = self.name\n        fields = {}\n        for f in self._FIELDS:\n            v = self._keyring.get(name, f)\n            if not self._is_field_optional(f):\n                while v is None:  # was not known\n                    v = self._ask_and_set(f)\n                fields[f] = v\n            elif v is not None:\n                fields[f] = v\n        return fields\n\n    def set(self, **kwargs):\n        \"\"\"Set field(s) of the credential\"\"\"\n        for f, v in kwargs.items():\n            if f not in self._FIELDS:\n                raise ValueError(\"Unknown field %s. Known are: %s\"\n                                 % (f, self._FIELDS.keys()))\n            self._keyring.set(self.name, f, v)\n\n    def get(self, f, default=None):\n        \"\"\"Get a field of the credential\"\"\"\n        if f not in self._FIELDS:\n            raise ValueError(\"Unknown field %s. Known are: %s\"\n                             % (f, self._FIELDS.keys()))\n        try:\n            return self._keyring.get(self.name, f)\n        except:  # MIH: what could even happen? _keyring not a dict?\n            return default\n\n    def delete(self):\n        \"\"\"Deletes credential values from the keyring\"\"\"\n        for f in self._FIELDS:\n            self._keyring.delete(self.name, f)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 39,
                            "end_lineno": 179,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/downloaders/credentials.py",
                            "type": "method",
                            "name": "Credential.enter_new",
                            "content": "class Credential(object):\n    \"\"\"Base class for different types of credentials\n    \"\"\"\n\n    # Should be defined in a subclass as an OrderedDict of fields\n    # name -> dict(attributes)\n    _FIELDS = None\n    _KNOWN_ATTRS = {\n        'hidden',    # UI should not display the value\n        'optional',  # Not mandatory thus not requested if not set\n    }\n\n    def __init__(self, name, url=None, keyring=None):\n        \"\"\"\n        Parameters\n        ----------\n        name : str\n            Name of the credential, as it would be identified by in the centralized\n            storage of credentials\n        url : str, optional\n            URL string to point users to where to seek obtaining the credentials\n        keyring : a keyring\n            An object providing (g|s)et_password.  If None, keyring module is used\n            as is\n        \"\"\"\n        self.name = name\n        self.url = url\n        self._keyring = keyring or keyring_\n        self._prepare()\n\n    def _prepare(self):\n        \"\"\"Additional house-keeping possibly to be performed in subclasses\n\n        Created to avoid all the passing args/kwargs of __init__ all the time\n        \"\"\"\n        # Basic checks\n        for f, fattrs in self._FIELDS.items():\n            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)\n            if unknown_attrs:\n                raise ValueError(\"Unknown attributes %s. Known are: %s\"\n                                 % (unknown_attrs, self._KNOWN_ATTRS))\n\n    def _is_field_optional(self, f):\n        return self._FIELDS[f].get('optional', False)\n\n    def _is_field_hidden(self, f):\n        return self._FIELDS[f].get('hidden', False)\n\n    @property\n    def is_known(self):\n        \"\"\"Return True if values for all fields of the credential are known\"\"\"\n        try:\n            return all(\n                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None\n                for f in self._FIELDS)\n        except Exception as exc:\n            lgr.warning(\"Failed to query keyring: %s\" % exc_str(exc))\n            return False\n\n    def _ask_field_value(self, f, instructions=None):\n        msg = instructions if instructions else \\\n            (\"You need to authenticate with %r credentials.\" % self.name +\n                  (\" %s provides information on how to gain access\"\n                   % self.url if self.url else ''))\n\n        return ui.question(\n            f,\n            title=msg,\n            hidden=self._is_field_hidden(f))\n\n    def _ask_and_set(self, f, instructions=None):\n        v = self._ask_field_value(f, instructions=instructions)\n        self.set(**{f: v})\n        return v\n\n    def enter_new(self, instructions=None, **kwargs):\n        \"\"\"Enter new values for the credential fields\n\n        Parameters\n        ----------\n        instructions : str, optional\n          If given, the auto-generated instructions based on a login-URL are\n          replaced by the given string\n        **kwargs\n          Any given key value pairs with non-None values are used to set the\n          field `key` to the given value, without asking for user input\n        \"\"\"\n        if kwargs:\n            unknown_fields = set(kwargs).difference(self._FIELDS)\n            known_fields = set(self._FIELDS).difference(kwargs)\n            if unknown_fields:\n                raise ValueError(\n                    \"Unknown to %s field(s): %s.  Known but not specified: %s\"\n                    % (self,\n                       ', '.join(sorted(unknown_fields)),\n                       ', '.join(sorted(known_fields))\n                       ))\n        # Use ui., request credential fields corresponding to the type\n        for f in self._FIELDS:\n            if kwargs.get(f, None):\n                # use given value, don't ask\n                self.set(**{f: kwargs[f]})\n            elif not self._is_field_optional(f):\n                self._ask_and_set(f, instructions=instructions)\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        name = self.name\n        fields = {}\n        for f in self._FIELDS:\n            v = self._keyring.get(name, f)\n            if not self._is_field_optional(f):\n                while v is None:  # was not known\n                    v = self._ask_and_set(f)\n                fields[f] = v\n            elif v is not None:\n                fields[f] = v\n        return fields\n\n    def set(self, **kwargs):\n        \"\"\"Set field(s) of the credential\"\"\"\n        for f, v in kwargs.items():\n            if f not in self._FIELDS:\n                raise ValueError(\"Unknown field %s. Known are: %s\"\n                                 % (f, self._FIELDS.keys()))\n            self._keyring.set(self.name, f, v)\n\n    def get(self, f, default=None):\n        \"\"\"Get a field of the credential\"\"\"\n        if f not in self._FIELDS:\n            raise ValueError(\"Unknown field %s. Known are: %s\"\n                             % (f, self._FIELDS.keys()))\n        try:\n            return self._keyring.get(self.name, f)\n        except:  # MIH: what could even happen? _keyring not a dict?\n            return default\n\n    def delete(self):\n        \"\"\"Deletes credential values from the keyring\"\"\"\n        for f in self._FIELDS:\n            self._keyring.delete(self.name, f)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 114,
                            "end_lineno": 142,
                            "parent": {
                                "file_name": "datalad/downloaders/credentials.py",
                                "type": "class",
                                "name": "Credential",
                                "content": "class Credential(object):\n    \"\"\"Base class for different types of credentials\n    \"\"\"\n\n    # Should be defined in a subclass as an OrderedDict of fields\n    # name -> dict(attributes)\n    _FIELDS = None\n    _KNOWN_ATTRS = {\n        'hidden',    # UI should not display the value\n        'optional',  # Not mandatory thus not requested if not set\n    }\n\n    def __init__(self, name, url=None, keyring=None):\n        \"\"\"\n        Parameters\n        ----------\n        name : str\n            Name of the credential, as it would be identified by in the centralized\n            storage of credentials\n        url : str, optional\n            URL string to point users to where to seek obtaining the credentials\n        keyring : a keyring\n            An object providing (g|s)et_password.  If None, keyring module is used\n            as is\n        \"\"\"\n        self.name = name\n        self.url = url\n        self._keyring = keyring or keyring_\n        self._prepare()\n\n    def _prepare(self):\n        \"\"\"Additional house-keeping possibly to be performed in subclasses\n\n        Created to avoid all the passing args/kwargs of __init__ all the time\n        \"\"\"\n        # Basic checks\n        for f, fattrs in self._FIELDS.items():\n            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)\n            if unknown_attrs:\n                raise ValueError(\"Unknown attributes %s. Known are: %s\"\n                                 % (unknown_attrs, self._KNOWN_ATTRS))\n\n    def _is_field_optional(self, f):\n        return self._FIELDS[f].get('optional', False)\n\n    def _is_field_hidden(self, f):\n        return self._FIELDS[f].get('hidden', False)\n\n    @property\n    def is_known(self):\n        \"\"\"Return True if values for all fields of the credential are known\"\"\"\n        try:\n            return all(\n                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None\n                for f in self._FIELDS)\n        except Exception as exc:\n            lgr.warning(\"Failed to query keyring: %s\" % exc_str(exc))\n            return False\n\n    def _ask_field_value(self, f, instructions=None):\n        msg = instructions if instructions else \\\n            (\"You need to authenticate with %r credentials.\" % self.name +\n                  (\" %s provides information on how to gain access\"\n                   % self.url if self.url else ''))\n\n        return ui.question(\n            f,\n            title=msg,\n            hidden=self._is_field_hidden(f))\n\n    def _ask_and_set(self, f, instructions=None):\n        v = self._ask_field_value(f, instructions=instructions)\n        self.set(**{f: v})\n        return v\n\n    def enter_new(self, instructions=None, **kwargs):\n        \"\"\"Enter new values for the credential fields\n\n        Parameters\n        ----------\n        instructions : str, optional\n          If given, the auto-generated instructions based on a login-URL are\n          replaced by the given string\n        **kwargs\n          Any given key value pairs with non-None values are used to set the\n          field `key` to the given value, without asking for user input\n        \"\"\"\n        if kwargs:\n            unknown_fields = set(kwargs).difference(self._FIELDS)\n            known_fields = set(self._FIELDS).difference(kwargs)\n            if unknown_fields:\n                raise ValueError(\n                    \"Unknown to %s field(s): %s.  Known but not specified: %s\"\n                    % (self,\n                       ', '.join(sorted(unknown_fields)),\n                       ', '.join(sorted(known_fields))\n                       ))\n        # Use ui., request credential fields corresponding to the type\n        for f in self._FIELDS:\n            if kwargs.get(f, None):\n                # use given value, don't ask\n                self.set(**{f: kwargs[f]})\n            elif not self._is_field_optional(f):\n                self._ask_and_set(f, instructions=instructions)\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        name = self.name\n        fields = {}\n        for f in self._FIELDS:\n            v = self._keyring.get(name, f)\n            if not self._is_field_optional(f):\n                while v is None:  # was not known\n                    v = self._ask_and_set(f)\n                fields[f] = v\n            elif v is not None:\n                fields[f] = v\n        return fields\n\n    def set(self, **kwargs):\n        \"\"\"Set field(s) of the credential\"\"\"\n        for f, v in kwargs.items():\n            if f not in self._FIELDS:\n                raise ValueError(\"Unknown field %s. Known are: %s\"\n                                 % (f, self._FIELDS.keys()))\n            self._keyring.set(self.name, f, v)\n\n    def get(self, f, default=None):\n        \"\"\"Get a field of the credential\"\"\"\n        if f not in self._FIELDS:\n            raise ValueError(\"Unknown field %s. Known are: %s\"\n                             % (f, self._FIELDS.keys()))\n        try:\n            return self._keyring.get(self.name, f)\n        except:  # MIH: what could even happen? _keyring not a dict?\n            return default\n\n    def delete(self):\n        \"\"\"Deletes credential values from the keyring\"\"\"\n        for f in self._FIELDS:\n            self._keyring.delete(self.name, f)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 39,
                                "end_lineno": 179,
                                "parent": null
                            }
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for credentials\"\"\"\n\nfrom mock import patch\n\nfrom datalad.tests.utils import with_testsui\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_true, assert_false\nfrom datalad.tests.utils import assert_raises\nfrom datalad.tests.utils import SkipTest\nfrom datalad.support.keyring_ import MemoryKeyring\nfrom datalad.support.keyring_ import Keyring\nfrom ..credentials import UserPassword\nfrom ..credentials import CompositeCredential\nfrom ..credentials import AWS_S3\n\n\n@with_testsui(responses=['user1', 'password1'])\ndef test_cred1_enter_new():\n    keyring = MemoryKeyring()\n    cred = UserPassword(\"name\", keyring=keyring)\n    assert_false(cred.is_known)\n    assert_equal(cred.enter_new(), None)\n    assert_true(cred.is_known)\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_equal(keyring.get('name', 'password'), 'password1')\n    keyring.delete('name')\n    assert_raises(KeyError, keyring.delete, 'name', 'user')\n    assert_raises(KeyError, keyring.delete, 'name')\n    assert_equal(keyring.get('name', 'user'), None)\n\n\n@with_testsui(responses=['password1'])\ndef test_cred1_call():\n    keyring = MemoryKeyring()\n    cred = UserPassword(\"name\", keyring=keyring)\n    # we will set the name but not the password, expecting UI\n    # requesting it\n    assert_equal(keyring.set('name', 'user', 'user1'), None)\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_equal(cred(), {'user': 'user1', 'password': 'password1'})\n    assert_equal(keyring.get('name', 'password'), 'password1')\n\n\ndef test_keyring():\n    # mock out keyring methods and test that we are providing correct values\n    # with 'datalad-' prefix\n    raise SkipTest(\"provide tests for Keyring which interfaces keyring module\")\n\n\ndef _cred1_adapter(composite, user=None, password=None):\n    \"\"\"Just a sample adapter from one user/pw type to another\"\"\"\n    return dict(user=user + \"_1\", password=password + \"_2\")\n\n\nclass _CCred1(CompositeCredential):\n    \"\"\"A Simple composite credential which will do some entries transformation\n    \"\"\"\n    _CREDENTIAL_CLASSES = (UserPassword, UserPassword)\n    _CREDENTIAL_ADAPTERS = (_cred1_adapter,)\n\n\n@with_testsui(responses=['user1', 'password1',\n                         'user2', 'password2'])\ndef test_composite_credential1():\n    # basic test of composite credential\n    keyring = MemoryKeyring()\n    cred = _CCred1(\"name\", keyring=keyring)\n    # When queried, does the chain\n    assert_equal(cred(), {'user': 'user1_1', 'password': 'password1_2'})\n    # But the \"Front\" credential is exposed to the user\n    assert_equal(cred.get('user'), 'user1')\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_raises(ValueError, cred.get, 'unknown_field')\n    assert_equal(cred.get('password'), 'password1')\n    assert_equal(keyring.get('name', 'password'), 'password1')\n    # ATM composite credential stores \"derived\" ones unconditionally in the\n    # keyring as well\n    assert_equal(keyring.get('name:1', 'user'), 'user1_1')\n    assert_equal(keyring.get('name:1', 'password'), 'password1_2')\n\n    # and now enter new should remove \"derived\" entries\n    cred.enter_new()\n    assert_equal(keyring.get('name', 'user'), 'user2')\n    assert_equal(keyring.get('name', 'password'), 'password2')\n    assert_equal(keyring.get('name:1', 'user'), None)\n    assert_equal(keyring.get('name:1', 'password'), None)\n    # which would get reevaluated if requested\n    assert_equal(keyring.entries, {'name:1': {}, 'name': {'user': 'user2', 'password': 'password2'}})\n    assert_equal(cred(), {'user': 'user2_1', 'password': 'password2_2'})\n\n\ndef test_credentials_from_env():\n    keyring = Keyring()\n    cred = AWS_S3(\"test-s3\", keyring=keyring)\n    assert_false(cred.is_known)\n    assert_equal(cred.get('key_id'), None)\n    assert_equal(cred.get('secret_id'), None)\n\n    with patch.dict('os.environ', {'DATALAD_test_s3_key_id': '1'}):\n        assert_equal(cred.get('key_id'), '1')\n        assert_false(cred.is_known)\n        with patch.dict('os.environ', {'DATALAD_test_s3_secret_id': '2'}):\n            assert_equal(cred.get('key_id'), '1')\n            assert_equal(cred.get('secret_id'), '2')\n            assert_true(cred.is_known)\n        assert_false(cred.is_known)  # no memory of the past",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for credentials\"\"\"\n\nfrom mock import patch\n\nfrom datalad.tests.utils import with_testsui\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_in\nfrom datalad.tests.utils import assert_true, assert_false\nfrom datalad.tests.utils import assert_raises\nfrom datalad.tests.utils import SkipTest\nfrom datalad.support.keyring_ import MemoryKeyring\nfrom datalad.support.keyring_ import Keyring\nfrom ..credentials import UserPassword\nfrom ..credentials import CompositeCredential\nfrom ..credentials import AWS_S3\n\n\n@with_testsui(responses=[\n    'user1', 'password1',\n    # when we do provide user to enter_new\n    'newpassword',\n])\ndef test_cred1_enter_new():\n    keyring = MemoryKeyring()\n    cred = UserPassword(\"name\", keyring=keyring)\n    assert_false(cred.is_known)\n    assert_equal(cred.enter_new(), None)\n    assert_true(cred.is_known)\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_equal(keyring.get('name', 'password'), 'password1')\n    keyring.delete('name')\n    assert_raises(KeyError, keyring.delete, 'name', 'user')\n    assert_raises(KeyError, keyring.delete, 'name')\n    assert_equal(keyring.get('name', 'user'), None)\n\n    # Test it blowing up if we provide unknown field\n    with assert_raises(ValueError) as cme:\n        cred.enter_new(username='user')\n    assert_in('field(s): username.  Known but not specified: password, user',\n              str(cme.exception))\n\n    # Test that if user is provided, it is not asked\n    cred.enter_new(user='user2')\n    assert_equal(keyring.get('name', 'user'), 'user2')\n    assert_equal(keyring.get('name', 'password'), 'newpassword')\n\n@with_testsui(responses=['password1'])\ndef test_cred1_call():\n    keyring = MemoryKeyring()\n    cred = UserPassword(\"name\", keyring=keyring)\n    # we will set the name but not the password, expecting UI\n    # requesting it\n    assert_equal(keyring.set('name', 'user', 'user1'), None)\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_equal(cred(), {'user': 'user1', 'password': 'password1'})\n    assert_equal(keyring.get('name', 'password'), 'password1')\n\n\ndef test_keyring():\n    # mock out keyring methods and test that we are providing correct values\n    # with 'datalad-' prefix\n    raise SkipTest(\"provide tests for Keyring which interfaces keyring module\")\n\n\ndef _cred1_adapter(composite, user=None, password=None):\n    \"\"\"Just a sample adapter from one user/pw type to another\"\"\"\n    return dict(user=user + \"_1\", password=password + \"_2\")\n\n\nclass _CCred1(CompositeCredential):\n    \"\"\"A Simple composite credential which will do some entries transformation\n    \"\"\"\n    _CREDENTIAL_CLASSES = (UserPassword, UserPassword)\n    _CREDENTIAL_ADAPTERS = (_cred1_adapter,)\n\n\n@with_testsui(responses=['user1', 'password1',\n                         'user2', 'password2'])\ndef test_composite_credential1():\n    # basic test of composite credential\n    keyring = MemoryKeyring()\n    cred = _CCred1(\"name\", keyring=keyring)\n    # When queried, does the chain\n    assert_equal(cred(), {'user': 'user1_1', 'password': 'password1_2'})\n    # But the \"Front\" credential is exposed to the user\n    assert_equal(cred.get('user'), 'user1')\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_raises(ValueError, cred.get, 'unknown_field')\n    assert_equal(cred.get('password'), 'password1')\n    assert_equal(keyring.get('name', 'password'), 'password1')\n    # ATM composite credential stores \"derived\" ones unconditionally in the\n    # keyring as well\n    assert_equal(keyring.get('name:1', 'user'), 'user1_1')\n    assert_equal(keyring.get('name:1', 'password'), 'password1_2')\n\n    # and now enter new should remove \"derived\" entries\n    cred.enter_new()\n    assert_equal(keyring.get('name', 'user'), 'user2')\n    assert_equal(keyring.get('name', 'password'), 'password2')\n    assert_equal(keyring.get('name:1', 'user'), None)\n    assert_equal(keyring.get('name:1', 'password'), None)\n    # which would get reevaluated if requested\n    assert_equal(keyring.entries, {'name:1': {}, 'name': {'user': 'user2', 'password': 'password2'}})\n    assert_equal(cred(), {'user': 'user2_1', 'password': 'password2_2'})\n\n\ndef test_credentials_from_env():\n    keyring = Keyring()\n    cred = AWS_S3(\"test-s3\", keyring=keyring)\n    assert_false(cred.is_known)\n    assert_equal(cred.get('key_id'), None)\n    assert_equal(cred.get('secret_id'), None)\n\n    with patch.dict('os.environ', {'DATALAD_test_s3_key_id': '1'}):\n        assert_equal(cred.get('key_id'), '1')\n        assert_false(cred.is_known)\n        with patch.dict('os.environ', {'DATALAD_test_s3_secret_id': '2'}):\n            assert_equal(cred.get('key_id'), '1')\n            assert_equal(cred.get('secret_id'), '2')\n            assert_true(cred.is_known)\n        assert_false(cred.is_known)  # no memory of the past",
            "header": {
                "file": {
                    "path": "datalad/downloaders/tests/test_credentials.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "c26cfa2a2",
                "new_commit_hash": "348c80510",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/downloaders/tests/test_credentials.py"
            },
            "plus_file": {
                "path": "b/datalad/downloaders/tests/test_credentials.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 12,
                            "length": 6
                        },
                        "new_range": {
                            "start": 12,
                            "length": 7
                        },
                        "section": "from mock import patch"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "from datalad.tests.utils import with_testsui",
                                "type": "context"
                            },
                            {
                                "content": "from datalad.tests.utils import assert_equal",
                                "type": "context"
                            },
                            {
                                "content": "from datalad.tests.utils import assert_in",
                                "type": "added"
                            },
                            {
                                "content": "from datalad.tests.utils import assert_true, assert_false",
                                "type": "context"
                            },
                            {
                                "content": "from datalad.tests.utils import assert_raises",
                                "type": "context"
                            },
                            {
                                "content": "from datalad.tests.utils import SkipTest",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_credentials.py",
                            "type": "statement",
                            "name": "",
                            "content": "from datalad.tests.utils import assert_in",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 15,
                            "end_lineno": 15,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 22,
                            "length": 7
                        },
                        "new_range": {
                            "start": 23,
                            "length": 11
                        },
                        "section": "from ..credentials import CompositeCredential"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "from ..credentials import AWS_S3",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@with_testsui(responses=['user1', 'password1'])",
                                "type": "deleted"
                            },
                            {
                                "content": "@with_testsui(responses=[",
                                "type": "added"
                            },
                            {
                                "content": "    'user1', 'password1',",
                                "type": "added"
                            },
                            {
                                "content": "    # when we do provide user to enter_new",
                                "type": "added"
                            },
                            {
                                "content": "    'newpassword',",
                                "type": "added"
                            },
                            {
                                "content": "])",
                                "type": "added"
                            },
                            {
                                "content": "def test_cred1_enter_new():",
                                "type": "context"
                            },
                            {
                                "content": "    keyring = MemoryKeyring()",
                                "type": "context"
                            },
                            {
                                "content": "    cred = UserPassword(\"name\", keyring=keyring)",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 36,
                            "length": 6
                        },
                        "new_range": {
                            "start": 41,
                            "length": 16
                        },
                        "section": "def test_cred1_enter_new():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    assert_raises(KeyError, keyring.delete, 'name')",
                                "type": "context"
                            },
                            {
                                "content": "    assert_equal(keyring.get('name', 'user'), None)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    # Test it blowing up if we provide unknown field",
                                "type": "added"
                            },
                            {
                                "content": "    with assert_raises(ValueError) as cme:",
                                "type": "added"
                            },
                            {
                                "content": "        cred.enter_new(username='user')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_in('field(s): username.  Known but not specified: password, user',",
                                "type": "added"
                            },
                            {
                                "content": "              str(cme.exception))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Test that if user is provided, it is not asked",
                                "type": "added"
                            },
                            {
                                "content": "    cred.enter_new(user='user2')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_equal(keyring.get('name', 'user'), 'user2')",
                                "type": "added"
                            },
                            {
                                "content": "    assert_equal(keyring.get('name', 'password'), 'newpassword')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "@with_testsui(responses=['password1'])",
                                "type": "context"
                            },
                            {
                                "content": "def test_cred1_call():",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/downloaders/tests/test_credentials.py",
                            "type": "function",
                            "name": "test_cred1_enter_new",
                            "content": "def test_cred1_enter_new():\n    keyring = MemoryKeyring()\n    cred = UserPassword(\"name\", keyring=keyring)\n    assert_false(cred.is_known)\n    assert_equal(cred.enter_new(), None)\n    assert_true(cred.is_known)\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_equal(keyring.get('name', 'password'), 'password1')\n    keyring.delete('name')\n    assert_raises(KeyError, keyring.delete, 'name', 'user')\n    assert_raises(KeyError, keyring.delete, 'name')\n    assert_equal(keyring.get('name', 'user'), None)\n\n    # Test it blowing up if we provide unknown field\n    with assert_raises(ValueError) as cme:\n        cred.enter_new(username='user')\n    assert_in('field(s): username.  Known but not specified: password, user',\n              str(cme.exception))\n\n    # Test that if user is provided, it is not asked\n    cred.enter_new(user='user2')\n    assert_equal(keyring.get('name', 'user'), 'user2')\n    assert_equal(keyring.get('name', 'password'), 'newpassword')",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 31,
                            "end_lineno": 53,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "2c4c6735e39077339e479aa3e7cc00719396631d^",
    "new_commit_hash": "2c4c6735e39077339e479aa3e7cc00719396631d",
    "commit_message": "ENH: verify that credential fields passed to enter_new are legit",
    "commit_date": "2019-04-23T16:09:59-04:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "2c4c6735e39077339e479aa3e7cc00719396631d",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for credentials\"\"\"\n\nfrom mock import patch\n\nfrom datalad.tests.utils import with_testsui\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_in\nfrom datalad.tests.utils import assert_true, assert_false\nfrom datalad.tests.utils import assert_raises\nfrom datalad.tests.utils import SkipTest\nfrom datalad.support.keyring_ import MemoryKeyring\nfrom datalad.support.keyring_ import Keyring\nfrom datalad.downloaders.credentials import UserPassword\nfrom datalad.downloaders.credentials import CompositeCredential\nfrom datalad.downloaders.credentials import AWS_S3\n\n\n@with_testsui(responses=[\n    'user1', 'password1',\n    # when we do provide user to enter_new\n    'newpassword',\n])\ndef test_cred1_enter_new():\n    keyring = MemoryKeyring()\n    cred = UserPassword(\"name\", keyring=keyring)\n    assert_false(cred.is_known)\n    assert_equal(cred.enter_new(), None)\n    assert_true(cred.is_known)\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_equal(keyring.get('name', 'password'), 'password1')\n    keyring.delete('name')\n    assert_raises(KeyError, keyring.delete, 'name', 'user')\n    assert_raises(KeyError, keyring.delete, 'name')\n    assert_equal(keyring.get('name', 'user'), None)\n\n    # Test it blowing up if we provide unknown field\n    with assert_raises(ValueError) as cme:\n        cred.enter_new(username='user')\n    assert_in('field(s): username.  Known but not specified: password, user',\n              str(cme.exception))\n\n    # Test that if user is provided, it is not asked\n    cred.enter_new(user='user2')\n    assert_equal(keyring.get('name', 'user'), 'user2')\n    assert_equal(keyring.get('name', 'password'), 'newpassword')\n\n@with_testsui(responses=['password1'])\ndef test_cred1_call():\n    keyring = MemoryKeyring()\n    cred = UserPassword(\"name\", keyring=keyring)\n    # we will set the name but not the password, expecting UI\n    # requesting it\n    assert_equal(keyring.set('name', 'user', 'user1'), None)\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_equal(cred(), {'user': 'user1', 'password': 'password1'})\n    assert_equal(keyring.get('name', 'password'), 'password1')\n\n\ndef test_keyring():\n    # mock out keyring methods and test that we are providing correct values\n    # with 'datalad-' prefix\n    raise SkipTest(\"provide tests for Keyring which interfaces keyring module\")\n\n\ndef _cred1_adapter(composite, user=None, password=None):\n    \"\"\"Just a sample adapter from one user/pw type to another\"\"\"\n    return dict(user=user + \"_1\", password=password + \"_2\")\n\n\nclass _CCred1(CompositeCredential):\n    \"\"\"A Simple composite credential which will do some entries transformation\n    \"\"\"\n    _CREDENTIAL_CLASSES = (UserPassword, UserPassword)\n    _CREDENTIAL_ADAPTERS = (_cred1_adapter,)\n\n\n@with_testsui(responses=['user1', 'password1',\n                         'user2', 'password2'])\ndef test_composite_credential1():\n    # basic test of composite credential\n    keyring = MemoryKeyring()\n    cred = _CCred1(\"name\", keyring=keyring)\n    # When queried, does the chain\n    assert_equal(cred(), {'user': 'user1_1', 'password': 'password1_2'})\n    # But the \"Front\" credential is exposed to the user\n    assert_equal(cred.get('user'), 'user1')\n    assert_equal(keyring.get('name', 'user'), 'user1')\n    assert_raises(ValueError, cred.get, 'unknown_field')\n    assert_equal(cred.get('password'), 'password1')\n    assert_equal(keyring.get('name', 'password'), 'password1')\n    # ATM composite credential stores \"derived\" ones unconditionally in the\n    # keyring as well\n    assert_equal(keyring.get('name:1', 'user'), 'user1_1')\n    assert_equal(keyring.get('name:1', 'password'), 'password1_2')\n\n    # and now enter new should remove \"derived\" entries\n    cred.enter_new()\n    assert_equal(keyring.get('name', 'user'), 'user2')\n    assert_equal(keyring.get('name', 'password'), 'password2')\n    assert_equal(keyring.get('name:1', 'user'), None)\n    assert_equal(keyring.get('name:1', 'password'), None)\n    # which would get reevaluated if requested\n    assert_equal(keyring.entries, {'name:1': {}, 'name': {'user': 'user2', 'password': 'password2'}})\n    assert_equal(cred(), {'user': 'user2_1', 'password': 'password2_2'})\n\n\ndef test_credentials_from_env():\n    keyring = Keyring()\n    cred = AWS_S3(\"test-s3\", keyring=keyring)\n    assert_false(cred.is_known)\n    assert_equal(cred.get('key_id'), None)\n    assert_equal(cred.get('secret_id'), None)\n\n    with patch.dict('os.environ', {'DATALAD_test_s3_key_id': '1'}):\n        assert_equal(cred.get('key_id'), '1')\n        assert_false(cred.is_known)\n        with patch.dict('os.environ', {'DATALAD_test_s3_secret_id': '2'}):\n            assert_equal(cred.get('key_id'), '1')\n            assert_equal(cred.get('secret_id'), '2')\n            assert_true(cred.is_known)\n        assert_false(cred.is_known)  # no memory of the past",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.11.3.dev22\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 84ms\nInstalled 25 packages in 6.24s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpRodmPg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpRodmPg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpRodmPg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpRodmPg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpRodmPg/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 51, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpdr1I5O/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpdr1I5O/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpdr1I5O/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpdr1I5O/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpdr1I5O/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 51, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2c4c6735e39077339e479aa3e7cc00719396631d/datalad/__init__.py\", line 43, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2c4c6735e39077339e479aa3e7cc00719396631d/datalad/cmd.py\", line 32, in <module>\n    from .support import path as op\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2c4c6735e39077339e479aa3e7cc00719396631d/datalad/support/path.py\", line 21, in <module>\n    from ..utils import (\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2c4c6735e39077339e479aa3e7cc00719396631d/datalad/utils.py\", line 26, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 234ms\nInstalled 26 packages in 3.10s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 57 packages in 1.32s\nPrepared 1 package in 1.11s\nInstalled 50 packages in 61ms\n + appdirs==1.4.4\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + cryptography==44.0.0\n + datalad==0.11.3 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2c4c6735e39077339e479aa3e7cc00719396631d)\n + deprecated==1.2.15\n + exifread==3.0.0\n + gitdb==4.0.11\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + idna==3.10\n + importlib-resources==5.12.0\n + iso8601==2.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack==1.0.5\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.12\n + pillow==9.5.0\n + pycparser==2.21\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + requests==2.31.0\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.1\n + soupsieve==2.4.1\n + tqdm==4.67.1\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\nResolved 91 packages in 203ms\nInstalled 31 packages in 108ms\n + alabaster==0.7.13\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.14.0\n + build==1.1.1\n + distlib==0.3.9\n + docutils==0.19\n + filelock==3.12.2\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.9.16\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + platformdirs==4.0.0\n + psutil==6.1.1\n + pygments==2.17.2\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n + tabulate==0.9.0\n + virtualenv==20.26.6\n",
    "new_commit_res_code": 0,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2c4c6735e39077339e479aa3e7cc00719396631d\nplugins: cov-4.1.0\ncollected 5 items\n\nr2e_tests/test_1.py ..s..                                                [100%]\n\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_cred1_enter_new\nPASSED r2e_tests/test_1.py::test_cred1_call\nPASSED r2e_tests/test_1.py::test_composite_credential1\nPASSED r2e_tests/test_1.py::test_credentials_from_env\nSKIPPED [1] .venv/lib/python3.7/site-packages/_pytest/unittest.py:371: provide tests for Keyring which interfaces keyring module\n========================= 4 passed, 1 skipped in 0.83s =========================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_2c4c6735e39077339e479aa3e7cc00719396631d\nplugins: cov-4.1.0\ncollected 5 items\n\nr2e_tests/test_1.py F.s..                                                [100%]\n\n=================================== FAILURES ===================================\n_____________________________ test_cred1_enter_new _____________________________\n\n    @with_testsui(responses=[\n        'user1', 'password1',\n        # when we do provide user to enter_new\n        'newpassword',\n    ])\n    def test_cred1_enter_new():\n        keyring = MemoryKeyring()\n        cred = UserPassword(\"name\", keyring=keyring)\n        assert_false(cred.is_known)\n        assert_equal(cred.enter_new(), None)\n        assert_true(cred.is_known)\n        assert_equal(keyring.get('name', 'user'), 'user1')\n        assert_equal(keyring.get('name', 'password'), 'password1')\n        keyring.delete('name')\n        assert_raises(KeyError, keyring.delete, 'name', 'user')\n        assert_raises(KeyError, keyring.delete, 'name')\n        assert_equal(keyring.get('name', 'user'), None)\n    \n        # Test it blowing up if we provide unknown field\n        with assert_raises(ValueError) as cme:\n>           cred.enter_new(username='user')\n\nr2e_tests/test_1.py:46: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/downloaders/credentials.py:132: in enter_new\n    self._ask_and_set(f, instructions=instructions)\ndatalad/downloaders/credentials.py:110: in _ask_and_set\n    v = self._ask_field_value(f, instructions=instructions)\ndatalad/downloaders/credentials.py:107: in _ask_field_value\n    hidden=self._is_field_hidden(f))\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def question(self, *args, **kwargs):\n        if not self._responses:\n            raise AssertionError(\n>               \"We are asked for a response whenever none is left to give\"\n            )\nE           AssertionError: We are asked for a response whenever none is left to give\n\ndatalad/ui/dialog.py:348: AssertionError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_cred1_call\nPASSED r2e_tests/test_1.py::test_composite_credential1\nPASSED r2e_tests/test_1.py::test_credentials_from_env\nSKIPPED [1] .venv/lib/python3.7/site-packages/_pytest/unittest.py:371: provide tests for Keyring which interfaces keyring module\nFAILED r2e_tests/test_1.py::test_cred1_enter_new - AssertionError: We are ask...\n==================== 1 failed, 3 passed, 1 skipped in 0.46s ====================\n",
    "old_commit_res_stderr": ""
}
datalad/downloaders/credentials.py datalad/downloaders/tests/test_credentials.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `enter_new` Method Accepts Invalid Credential Fields Without Raising Errors

**Description:**
When using the `enter_new` method to add new credentials, providing fields that are not recognized by the system does not trigger any errors. This oversight allows the inclusion of unintended fields, which can lead to unexpected behavior and hinder proper credential management.

**Example Code:**
```python
keyring = MemoryKeyring()
cred = UserPassword("name", keyring=keyring)

# Attempting to add a credential with an invalid field 'username'
cred.enter_new(username='user')
```

**Expected Behavior:**
The `enter_new` method should validate all provided fields against the accepted credential fields. If any unknown or invalid fields are detected, the method should raise a `ValueError` clearly indicating which fields are unrecognized and listing the valid fields that were not specified.

**Actual Behavior:**
Providing an invalid field like `username` does not result in an error. Instead, the method proceeds without validation, which can cause issues such as missing required fields or inconsistent credential states. This lack of validation was evidenced by test failures where an assertion error was raised due to unexpected prompts for user input when no responses were left.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/downloaders/credentials.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface information about credentials

Provides minimalistic interface to deal (query, request, store) with most common
types of credentials.  To be used by Authenticators
"""

__dev_doc__ = """
Possibly useful in the future 3rd part developments

https://github.com/omab/python-social-auth
    social authentication/registration mechanism with support for several
    frameworks and auth providers
"""

import time
import calendar

from collections import OrderedDict

from ..dochelpers import exc_str
from ..support.keyring_ import keyring as keyring_
from ..ui import ui
from ..utils import auto_repr
from ..support.network import iso8601_to_epoch

from logging import getLogger
lgr = getLogger('datalad.downloaders.credentials')


@auto_repr
class Credential(object):
    """Base class for different types of credentials
    """

    # Should be defined in a subclass as an OrderedDict of fields
    # name -> dict(attributes)
    _FIELDS = None
    _KNOWN_ATTRS = {
        'hidden',    # UI should not display the value
        'optional',  # Not mandatory thus not requested if not set
    }

    def __init__(self, name, url=None, keyring=None):
        """
        Parameters
        ----------
        name : str
            Name of the credential, as it would be identified by in the centralized
            storage of credentials
        url : str, optional
            URL string to point users to where to seek obtaining the credentials
        keyring : a keyring
            An object providing (g|s)et_password.  If None, keyring module is used
            as is
        """
        self.name = name
        self.url = url
        self._keyring = keyring or keyring_
        self._prepare()

    def _prepare(self):
        """Additional house-keeping possibly to be performed in subclasses

        Created to avoid all the passing args/kwargs of __init__ all the time
        """
        # Basic checks
        for f, fattrs in self._FIELDS.items():
            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)
            if unknown_attrs:
                raise ValueError("Unknown attributes %s. Known are: %s"
                                 % (unknown_attrs, self._KNOWN_ATTRS))

    def _is_field_optional(self, f):
        return self._FIELDS[f].get('optional', False)

    def _is_field_hidden(self, f):
        return self._FIELDS[f].get('hidden', False)

    @property
    def is_known(self):
        """Return True if values for all fields of the credential are known"""
        try:
            return all(
                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None
                for f in self._FIELDS)
        except Exception as exc:
            lgr.warning("Failed to query keyring: %s" % exc_str(exc))
            return False

    def _ask_field_value(self, f, instructions=None):
        msg = instructions if instructions else \
            ("You need to authenticate with %r credentials." % self.name +
                  (" %s provides information on how to gain access"
                   % self.url if self.url else ''))

        return ui.question(
            f,
            title=msg,
            hidden=self._is_field_hidden(f))

    def _ask_and_set(self, f, instructions=None):
        v = self._ask_field_value(f, instructions=instructions)
        self.set(**{f: v})
        return v

    def enter_new(self, instructions=None, **kwargs):
        """Enter new values for the credential fields

        Parameters
        ----------
        instructions : str, optional
          If given, the auto-generated instructions based on a login-URL are
          replaced by the given string
        **kwargs
          Any given key value pairs with non-None values are used to set the
          field `key` to the given value, without asking for user input
        """
        # Use ui., request credential fields corresponding to the type
        for f in self._FIELDS:
            if kwargs.get(f, None):
                # use given value, don't ask
                self.set(**{f: kwargs[f]})
            elif not self._is_field_optional(f):
                self._ask_and_set(f, instructions=instructions)

    def __call__(self):
        """Obtain credentials from a keyring and if any is not known -- ask"""
        name = self.name
        fields = {}
        for f in self._FIELDS:
            v = self._keyring.get(name, f)
            if not self._is_field_optional(f):
                while v is None:  # was not known
                    v = self._ask_and_set(f)
                fields[f] = v
            elif v is not None:
                fields[f] = v
        return fields

    def set(self, **kwargs):
        """Set field(s) of the credential"""
        for f, v in kwargs.items():
            if f not in self._FIELDS:
                raise ValueError("Unknown field %s. Known are: %s"
                                 % (f, self._FIELDS.keys()))
            self._keyring.set(self.name, f, v)

    def get(self, f, default=None):
        """Get a field of the credential"""
        if f not in self._FIELDS:
            raise ValueError("Unknown field %s. Known are: %s"
                             % (f, self._FIELDS.keys()))
        try:
            return self._keyring.get(self.name, f)
        except:  # MIH: what could even happen? _keyring not a dict?
            return default

    def delete(self):
        """Deletes credential values from the keyring"""
        for f in self._FIELDS:
            self._keyring.delete(self.name, f)


class UserPassword(Credential):
    """Simple type of a credential which consists of user/password pair"""

    _FIELDS = OrderedDict([('user', {}),
                           ('password', {'hidden': True})])

    is_expired = False  # no expiration provisioned


class Token(Credential):
    """Simple type of a credential which provides a single token"""

    _FIELDS = OrderedDict([('token', {'hidden': True})])

    is_expired = False  # no expiration provisioned


class AWS_S3(Credential):
    """Credential for AWS S3 service"""

    _FIELDS = OrderedDict([('key_id', {}),
                           ('secret_id', {'hidden': True}),
                           ('session', {'optional': True}),
                           ('expiration', {'optional': True}),
                           ])

    @property
    def is_expired(self):
        exp = self.get('expiration', None)
        if not exp:
            return True
        exp_epoch = iso8601_to_epoch(exp)
        expire_in = (exp_epoch - calendar.timegm(time.localtime())) / 3600.

        lgr.debug(
            ("Credential %s has expired %.2fh ago"
                if expire_in <= 0 else "Credential %s will expire in %.2fh")
            % (self.name, expire_in))
        return expire_in <= 0


@auto_repr
class CompositeCredential(Credential):
    """Credential which represent a sequence of Credentials where front one is exposed to user
    """

    # To be defined in sub-classes
    _CREDENTIAL_CLASSES = None
    _CREDENTIAL_ADAPTERS = None

    def _prepare(self):
        assert len(self._CREDENTIAL_CLASSES) > 1, "makes sense only if there is > 1 credential"
        assert len(self._CREDENTIAL_CLASSES) == len(self._CREDENTIAL_ADAPTERS) + 1, \
            "there should be 1 less of adapter than _CREDENTIAL_CLASSES"

        for C in self._CREDENTIAL_CLASSES:
            assert issubclass(C, Credential), "%s must be a subclass of Credential" % C

        # First credential should bear the name and url
        credentials = [self._CREDENTIAL_CLASSES[0](self.name, url=self.url, keyring=self._keyring)]
        # and we just reuse its _FIELDS for _ask_field_value etc
        self._FIELDS = credentials[0]._FIELDS
        # the rest with index suffix, but storing themselves in the same keyring
        for iC, C in enumerate(self._CREDENTIAL_CLASSES[1:]):
            credentials.append(
                C(name="%s:%d" % (self.name, iC + 1), url=None, keyring=self._keyring)
            )
        self._credentials = credentials

        super(CompositeCredential, self)._prepare()

    # Here it becomes tricky, since theoretically it is the "tail"
    # ones which might expire etc, so we wouldn't exactly know what
    # new credentials outside process wanted -- it would be silly to ask
    # right away the "entry" credentials if it is just the expiration of the
    # tail credentials
    def enter_new(self):
        # should invalidate/remove all tail credentials to avoid failing attempts to login
        self._credentials[0].enter_new()
        for c in self._credentials[1:]:
            c.delete()

    def __call__(self):
        """Obtain credentials from a keyring and if any is not known -- ask"""
        # Start from the tail until we have credentials set
        idx = len(self._credentials) - 1
        for c in self._credentials[::-1]:
            if c.is_known and not c.is_expired:
                break
            idx -= 1

        if idx < 0:
            # none was known, all the same -- start with the first one
            idx = 0

        # TODO: consider moving logic of traversal into authenticator since it is
        # the one spitting out authentication error etc
        # Theoretically we could just reuse 'fields' from adapter in the next step
        # but let's do full circle, so that if any "normalization" is done by
        # Credentials we take that into account
        for c, adapter, next_c in zip(
                self._credentials[idx:],
                self._CREDENTIAL_ADAPTERS[idx:],
                self._credentials[idx + 1:]):
            fields = c()
            next_fields = adapter(self, **fields)
            next_c.set(**next_fields)

        return self._credentials[-1]()


def _nda_adapter(composite, user=None, password=None):
    from datalad.support.third.nda_aws_token_generator import NDATokenGenerator
    gen = NDATokenGenerator()
    token = gen.generate_token(user, password)
    # There are also session and expiration we ignore... TODO anything about it?!!!
    # we could create a derived AWS_S3 which would also store session and expiration
    # and then may be Composite could use those????
    return dict(key_id=token.access_key, secret_id=token.secret_key,
                session=token.session, expiration=token.expiration)


class NDA_S3(CompositeCredential):
    """Credential to access NDA AWS

    So for NDA we need a credential which is a composite credential.
    User provides UserPassword and then some adapter generates AWS_S3
    out of it
    """
    _CREDENTIAL_CLASSES = (UserPassword, AWS_S3)
    _CREDENTIAL_ADAPTERS = (_nda_adapter,)


def _loris_adapter(composite, user=None, password=None, **kwargs):
    from datalad.support.third.loris_token_generator import LORISTokenGenerator

    gen = LORISTokenGenerator(url=composite.url)
    token = gen.generate_token(user, password)

    return dict(token=token)


class LORIS_Token(CompositeCredential):
    _CREDENTIAL_CLASSES = (UserPassword, Token)
    _CREDENTIAL_ADAPTERS = (_loris_adapter,)

    def __init__(self, name, url=None, keyring=None):
        super(CompositeCredential, self).__init__(name, url, keyring)


CREDENTIAL_TYPES = {
    'user_password': UserPassword,
    'aws-s3': AWS_S3,
    'nda-s3': NDA_S3,
    'token': Token,
    'loris-token': LORIS_Token
}

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** `enter_new` Method Accepts Invalid Credential Fields Without Raising Errors

**Description:**
When using the `enter_new` method to add new credentials, providing fields that are not recognized by the system does not trigger any errors. This oversight allows the inclusion of unintended fields, which can lead to unexpected behavior and hinder proper credential management.

**Example Code:**
```python
keyring = MemoryKeyring()
cred = UserPassword("name", keyring=keyring)

# Attempting to add a credential with an invalid field 'username'
cred.enter_new(username='user')
```

**Expected Behavior:**
The `enter_new` method should validate all provided fields against the accepted credential fields. If any unknown or invalid fields are detected, the method should raise a `ValueError` clearly indicating which fields are unrecognized and listing the valid fields that were not specified.

**Actual Behavior:**
Providing an invalid field like `username` does not result in an error. Instead, the method proceeds without validation, which can cause issues such as missing required fields or inconsistent credential states. This lack of validation was evidenced by test failures where an assertion error was raised due to unexpected prompts for user input when no responses were left.

[/ISSUE]
{
    "test_cred1_enter_new": "PASSED",
    "test_cred1_call": "PASSED",
    "test_composite_credential1": "PASSED",
    "test_credentials_from_env": "PASSED"
}
diff --git a/datalad/downloaders/credentials.py b/datalad/downloaders/credentials.py
index a486c9c92..22446eed1 100644
--- a/datalad/downloaders/credentials.py
+++ b/datalad/downloaders/credentials.py
@@ -123,6 +123,16 @@ class Credential(object):
           Any given key value pairs with non-None values are used to set the
           field `key` to the given value, without asking for user input
         """
+        if kwargs:
+            unknown_fields = set(kwargs).difference(self._FIELDS)
+            known_fields = set(self._FIELDS).difference(kwargs)
+            if unknown_fields:
+                raise ValueError(
+                    "Unknown to %s field(s): %s.  Known but not specified: %s"
+                    % (self,
+                       ', '.join(sorted(unknown_fields)),
+                       ', '.join(sorted(known_fields))
+                       ))
         # Use ui., request credential fields corresponding to the type
         for f in self._FIELDS:
             if kwargs.get(f, None):

{"datalad/downloaders/credentials.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface information about credentials\n\nProvides minimalistic interface to deal (query, request, store) with most common\ntypes of credentials.  To be used by Authenticators\n\"\"\"\n\n__dev_doc__ = \"\"\"\nPossibly useful in the future 3rd part developments\n\nhttps://github.com/omab/python-social-auth\n    social authentication/registration mechanism with support for several\n    frameworks and auth providers\n\"\"\"\n\nimport time\nimport calendar\n\nfrom collections import OrderedDict\n\nfrom ..dochelpers import exc_str\nfrom ..support.keyring_ import keyring as keyring_\nfrom ..ui import ui\nfrom ..utils import auto_repr\nfrom ..support.network import iso8601_to_epoch\n\nfrom logging import getLogger\nlgr = getLogger('datalad.downloaders.credentials')\n\n\n@auto_repr\nclass Credential(object):\n    \"\"\"Base class for different types of credentials\n    \"\"\"\n\n    # Should be defined in a subclass as an OrderedDict of fields\n    # name -> dict(attributes)\n    _FIELDS = None\n    _KNOWN_ATTRS = {\n        'hidden',    # UI should not display the value\n        'optional',  # Not mandatory thus not requested if not set\n    }\n\n    def __init__(self, name, url=None, keyring=None):\n        \"\"\"\n        Parameters\n        ----------\n        name : str\n            Name of the credential, as it would be identified by in the centralized\n            storage of credentials\n        url : str, optional\n            URL string to point users to where to seek obtaining the credentials\n        keyring : a keyring\n            An object providing (g|s)et_password.  If None, keyring module is used\n            as is\n        \"\"\"\n        self.name = name\n        self.url = url\n        self._keyring = keyring or keyring_\n        self._prepare()\n\n    def _prepare(self):\n        \"\"\"Additional house-keeping possibly to be performed in subclasses\n\n        Created to avoid all the passing args/kwargs of __init__ all the time\n        \"\"\"\n        # Basic checks\n        for f, fattrs in self._FIELDS.items():\n            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)\n            if unknown_attrs:\n                raise ValueError(\"Unknown attributes %s. Known are: %s\"\n                                 % (unknown_attrs, self._KNOWN_ATTRS))\n\n    def _is_field_optional(self, f):\n        return self._FIELDS[f].get('optional', False)\n\n    def _is_field_hidden(self, f):\n        return self._FIELDS[f].get('hidden', False)\n\n    @property\n    def is_known(self):\n        \"\"\"Return True if values for all fields of the credential are known\"\"\"\n        try:\n            return all(\n                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None\n                for f in self._FIELDS)\n        except Exception as exc:\n            lgr.warning(\"Failed to query keyring: %s\" % exc_str(exc))\n            return False\n\n    def _ask_field_value(self, f, instructions=None):\n        msg = instructions if instructions else \\\n            (\"You need to authenticate with %r credentials.\" % self.name +\n                  (\" %s provides information on how to gain access\"\n                   % self.url if self.url else ''))\n\n        return ui.question(\n            f,\n            title=msg,\n            hidden=self._is_field_hidden(f))\n\n    def _ask_and_set(self, f, instructions=None):\n        v = self._ask_field_value(f, instructions=instructions)\n        self.set(**{f: v})\n        return v\n\n    def enter_new(self, instructions=None, **kwargs):\n        \"\"\"Enter new values for the credential fields\n\n        Parameters\n        ----------\n        instructions : str, optional\n          If given, the auto-generated instructions based on a login-URL are\n          replaced by the given string\n        **kwargs\n          Any given key value pairs with non-None values are used to set the\n          field `key` to the given value, without asking for user input\n        \"\"\"\n        # Use ui., request credential fields corresponding to the type\n        for f in self._FIELDS:\n            if kwargs.get(f, None):\n                # use given value, don't ask\n                self.set(**{f: kwargs[f]})\n            elif not self._is_field_optional(f):\n                self._ask_and_set(f, instructions=instructions)\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        name = self.name\n        fields = {}\n        for f in self._FIELDS:\n            v = self._keyring.get(name, f)\n            if not self._is_field_optional(f):\n                while v is None:  # was not known\n                    v = self._ask_and_set(f)\n                fields[f] = v\n            elif v is not None:\n                fields[f] = v\n        return fields\n\n    def set(self, **kwargs):\n        \"\"\"Set field(s) of the credential\"\"\"\n        for f, v in kwargs.items():\n            if f not in self._FIELDS:\n                raise ValueError(\"Unknown field %s. Known are: %s\"\n                                 % (f, self._FIELDS.keys()))\n            self._keyring.set(self.name, f, v)\n\n    def get(self, f, default=None):\n        \"\"\"Get a field of the credential\"\"\"\n        if f not in self._FIELDS:\n            raise ValueError(\"Unknown field %s. Known are: %s\"\n                             % (f, self._FIELDS.keys()))\n        try:\n            return self._keyring.get(self.name, f)\n        except:  # MIH: what could even happen? _keyring not a dict?\n            return default\n\n    def delete(self):\n        \"\"\"Deletes credential values from the keyring\"\"\"\n        for f in self._FIELDS:\n            self._keyring.delete(self.name, f)\n\n\nclass UserPassword(Credential):\n    \"\"\"Simple type of a credential which consists of user/password pair\"\"\"\n\n    _FIELDS = OrderedDict([('user', {}),\n                           ('password', {'hidden': True})])\n\n    is_expired = False  # no expiration provisioned\n\n\nclass Token(Credential):\n    \"\"\"Simple type of a credential which provides a single token\"\"\"\n\n    _FIELDS = OrderedDict([('token', {'hidden': True})])\n\n    is_expired = False  # no expiration provisioned\n\n\nclass AWS_S3(Credential):\n    \"\"\"Credential for AWS S3 service\"\"\"\n\n    _FIELDS = OrderedDict([('key_id', {}),\n                           ('secret_id', {'hidden': True}),\n                           ('session', {'optional': True}),\n                           ('expiration', {'optional': True}),\n                           ])\n\n    @property\n    def is_expired(self):\n        exp = self.get('expiration', None)\n        if not exp:\n            return True\n        exp_epoch = iso8601_to_epoch(exp)\n        expire_in = (exp_epoch - calendar.timegm(time.localtime())) / 3600.\n\n        lgr.debug(\n            (\"Credential %s has expired %.2fh ago\"\n                if expire_in <= 0 else \"Credential %s will expire in %.2fh\")\n            % (self.name, expire_in))\n        return expire_in <= 0\n\n\n@auto_repr\nclass CompositeCredential(Credential):\n    \"\"\"Credential which represent a sequence of Credentials where front one is exposed to user\n    \"\"\"\n\n    # To be defined in sub-classes\n    _CREDENTIAL_CLASSES = None\n    _CREDENTIAL_ADAPTERS = None\n\n    def _prepare(self):\n        assert len(self._CREDENTIAL_CLASSES) > 1, \"makes sense only if there is > 1 credential\"\n        assert len(self._CREDENTIAL_CLASSES) == len(self._CREDENTIAL_ADAPTERS) + 1, \\\n            \"there should be 1 less of adapter than _CREDENTIAL_CLASSES\"\n\n        for C in self._CREDENTIAL_CLASSES:\n            assert issubclass(C, Credential), \"%s must be a subclass of Credential\" % C\n\n        # First credential should bear the name and url\n        credentials = [self._CREDENTIAL_CLASSES[0](self.name, url=self.url, keyring=self._keyring)]\n        # and we just reuse its _FIELDS for _ask_field_value etc\n        self._FIELDS = credentials[0]._FIELDS\n        # the rest with index suffix, but storing themselves in the same keyring\n        for iC, C in enumerate(self._CREDENTIAL_CLASSES[1:]):\n            credentials.append(\n                C(name=\"%s:%d\" % (self.name, iC + 1), url=None, keyring=self._keyring)\n            )\n        self._credentials = credentials\n\n        super(CompositeCredential, self)._prepare()\n\n    # Here it becomes tricky, since theoretically it is the \"tail\"\n    # ones which might expire etc, so we wouldn't exactly know what\n    # new credentials outside process wanted -- it would be silly to ask\n    # right away the \"entry\" credentials if it is just the expiration of the\n    # tail credentials\n    def enter_new(self):\n        # should invalidate/remove all tail credentials to avoid failing attempts to login\n        self._credentials[0].enter_new()\n        for c in self._credentials[1:]:\n            c.delete()\n\n    def __call__(self):\n        \"\"\"Obtain credentials from a keyring and if any is not known -- ask\"\"\"\n        # Start from the tail until we have credentials set\n        idx = len(self._credentials) - 1\n        for c in self._credentials[::-1]:\n            if c.is_known and not c.is_expired:\n                break\n            idx -= 1\n\n        if idx < 0:\n            # none was known, all the same -- start with the first one\n            idx = 0\n\n        # TODO: consider moving logic of traversal into authenticator since it is\n        # the one spitting out authentication error etc\n        # Theoretically we could just reuse 'fields' from adapter in the next step\n        # but let's do full circle, so that if any \"normalization\" is done by\n        # Credentials we take that into account\n        for c, adapter, next_c in zip(\n                self._credentials[idx:],\n                self._CREDENTIAL_ADAPTERS[idx:],\n                self._credentials[idx + 1:]):\n            fields = c()\n            next_fields = adapter(self, **fields)\n            next_c.set(**next_fields)\n\n        return self._credentials[-1]()\n\n\ndef _nda_adapter(composite, user=None, password=None):\n    from datalad.support.third.nda_aws_token_generator import NDATokenGenerator\n    gen = NDATokenGenerator()\n    token = gen.generate_token(user, password)\n    # There are also session and expiration we ignore... TODO anything about it?!!!\n    # we could create a derived AWS_S3 which would also store session and expiration\n    # and then may be Composite could use those????\n    return dict(key_id=token.access_key, secret_id=token.secret_key,\n                session=token.session, expiration=token.expiration)\n\n\nclass NDA_S3(CompositeCredential):\n    \"\"\"Credential to access NDA AWS\n\n    So for NDA we need a credential which is a composite credential.\n    User provides UserPassword and then some adapter generates AWS_S3\n    out of it\n    \"\"\"\n    _CREDENTIAL_CLASSES = (UserPassword, AWS_S3)\n    _CREDENTIAL_ADAPTERS = (_nda_adapter,)\n\n\ndef _loris_adapter(composite, user=None, password=None, **kwargs):\n    from datalad.support.third.loris_token_generator import LORISTokenGenerator\n\n    gen = LORISTokenGenerator(url=composite.url)\n    token = gen.generate_token(user, password)\n\n    return dict(token=token)\n\n\nclass LORIS_Token(CompositeCredential):\n    _CREDENTIAL_CLASSES = (UserPassword, Token)\n    _CREDENTIAL_ADAPTERS = (_loris_adapter,)\n\n    def __init__(self, name, url=None, keyring=None):\n        super(CompositeCredential, self).__init__(name, url, keyring)\n\n\nCREDENTIAL_TYPES = {\n    'user_password': UserPassword,\n    'aws-s3': AWS_S3,\n    'nda-s3': NDA_S3,\n    'token': Token,\n    'loris-token': LORIS_Token\n}\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `enter_new` Method Accepts Invalid Credential Fields Without Raising Errors

**Description:**
When using the `enter_new` method to add new credentials, providing fields that are not recognized by the system does not trigger any errors. This oversight allows the inclusion of unintended fields, which can lead to unexpected behavior and hinder proper credential management.

**Example Code:**
```python
keyring = MemoryKeyring()
cred = UserPassword("name", keyring=keyring)

# Attempting to add a credential with an invalid field 'username'
cred.enter_new(username='user')
```

**Expected Behavior:**
The `enter_new` method should validate all provided fields against the accepted credential fields. If any unknown or invalid fields are detected, the method should raise a `ValueError` clearly indicating which fields are unrecognized and listing the valid fields that were not specified.

**Actual Behavior:**
Providing an invalid field like `username` does not result in an error. Instead, the method proceeds without validation, which can cause issues such as missing required fields or inconsistent credential states. This lack of validation was evidenced by test failures where an assertion error was raised due to unexpected prompts for user input when no responses were left.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/downloaders/credentials.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface information about credentials

Provides minimalistic interface to deal (query, request, store) with most common
types of credentials.  To be used by Authenticators
"""

__dev_doc__ = """
Possibly useful in the future 3rd part developments

https://github.com/omab/python-social-auth
    social authentication/registration mechanism with support for several
    frameworks and auth providers
"""

import time
import calendar

from collections import OrderedDict

from ..dochelpers import exc_str
from ..support.keyring_ import keyring as keyring_
from ..ui import ui
from ..utils import auto_repr
from ..support.network import iso8601_to_epoch

from logging import getLogger
lgr = getLogger('datalad.downloaders.credentials')


@auto_repr
class Credential(object):
    """Base class for different types of credentials
    """

    # Should be defined in a subclass as an OrderedDict of fields
    # name -> dict(attributes)
    _FIELDS = None
    _KNOWN_ATTRS = {
        'hidden',    # UI should not display the value
        'optional',  # Not mandatory thus not requested if not set
    }

    def __init__(self, name, url=None, keyring=None):
        """
        Parameters
        ----------
        name : str
            Name of the credential, as it would be identified by in the centralized
            storage of credentials
        url : str, optional
            URL string to point users to where to seek obtaining the credentials
        keyring : a keyring
            An object providing (g|s)et_password.  If None, keyring module is used
            as is
        """
        self.name = name
        self.url = url
        self._keyring = keyring or keyring_
        self._prepare()

    def _prepare(self):
        """Additional house-keeping possibly to be performed in subclasses

        Created to avoid all the passing args/kwargs of __init__ all the time
        """
        # Basic checks
        for f, fattrs in self._FIELDS.items():
            unknown_attrs = set(fattrs).difference(self._KNOWN_ATTRS)
            if unknown_attrs:
                raise ValueError("Unknown attributes %s. Known are: %s"
                                 % (unknown_attrs, self._KNOWN_ATTRS))

    def _is_field_optional(self, f):
        return self._FIELDS[f].get('optional', False)

    def _is_field_hidden(self, f):
        return self._FIELDS[f].get('hidden', False)

    @property
    def is_known(self):
        """Return True if values for all fields of the credential are known"""
        try:
            return all(
                self._is_field_optional(f) or self._keyring.get(self.name, f) is not None
                for f in self._FIELDS)
        except Exception as exc:
            lgr.warning("Failed to query keyring: %s" % exc_str(exc))
            return False

    def _ask_field_value(self, f, instructions=None):
        msg = instructions if instructions else \
            ("You need to authenticate with %r credentials." % self.name +
                  (" %s provides information on how to gain access"
                   % self.url if self.url else ''))

        return ui.question(
            f,
            title=msg,
            hidden=self._is_field_hidden(f))

    def _ask_and_set(self, f, instructions=None):
        v = self._ask_field_value(f, instructions=instructions)
        self.set(**{f: v})
        return v

    def enter_new(self, instructions=None, **kwargs):
        """Enter new values for the credential fields

        Parameters
        ----------
        instructions : str, optional
          If given, the auto-generated instructions based on a login-URL are
          replaced by the given string
        **kwargs
          Any given key value pairs with non-None values are used to set the
          field `key` to the given value, without asking for user input
        """
        # Use ui., request credential fields corresponding to the type
        for f in self._FIELDS:
            if kwargs.get(f, None):
                # use given value, don't ask
                self.set(**{f: kwargs[f]})
            elif not self._is_field_optional(f):
                self._ask_and_set(f, instructions=instructions)

    def __call__(self):
        """Obtain credentials from a keyring and if any is not known -- ask"""
        name = self.name
        fields = {}
        for f in self._FIELDS:
            v = self._keyring.get(name, f)
            if not self._is_field_optional(f):
                while v is None:  # was not known
                    v = self._ask_and_set(f)
                fields[f] = v
            elif v is not None:
                fields[f] = v
        return fields

    def set(self, **kwargs):
        """Set field(s) of the credential"""
        for f, v in kwargs.items():
            if f not in self._FIELDS:
                raise ValueError("Unknown field %s. Known are: %s"
                                 % (f, self._FIELDS.keys()))
            self._keyring.set(self.name, f, v)

    def get(self, f, default=None):
        """Get a field of the credential"""
        if f not in self._FIELDS:
            raise ValueError("Unknown field %s. Known are: %s"
                             % (f, self._FIELDS.keys()))
        try:
            return self._keyring.get(self.name, f)
        except:  # MIH: what could even happen? _keyring not a dict?
            return default

    def delete(self):
        """Deletes credential values from the keyring"""
        for f in self._FIELDS:
            self._keyring.delete(self.name, f)


class UserPassword(Credential):
    """Simple type of a credential which consists of user/password pair"""

    _FIELDS = OrderedDict([('user', {}),
                           ('password', {'hidden': True})])

    is_expired = False  # no expiration provisioned


class Token(Credential):
    """Simple type of a credential which provides a single token"""

    _FIELDS = OrderedDict([('token', {'hidden': True})])

    is_expired = False  # no expiration provisioned


class AWS_S3(Credential):
    """Credential for AWS S3 service"""

    _FIELDS = OrderedDict([('key_id', {}),
                           ('secret_id', {'hidden': True}),
                           ('session', {'optional': True}),
                           ('expiration', {'optional': True}),
                           ])

    @property
    def is_expired(self):
        exp = self.get('expiration', None)
        if not exp:
            return True
        exp_epoch = iso8601_to_epoch(exp)
        expire_in = (exp_epoch - calendar.timegm(time.localtime())) / 3600.

        lgr.debug(
            ("Credential %s has expired %.2fh ago"
                if expire_in <= 0 else "Credential %s will expire in %.2fh")
            % (self.name, expire_in))
        return expire_in <= 0


@auto_repr
class CompositeCredential(Credential):
    """Credential which represent a sequence of Credentials where front one is exposed to user
    """

    # To be defined in sub-classes
    _CREDENTIAL_CLASSES = None
    _CREDENTIAL_ADAPTERS = None

    def _prepare(self):
        assert len(self._CREDENTIAL_CLASSES) > 1, "makes sense only if there is > 1 credential"
        assert len(self._CREDENTIAL_CLASSES) == len(self._CREDENTIAL_ADAPTERS) + 1, \
            "there should be 1 less of adapter than _CREDENTIAL_CLASSES"

        for C in self._CREDENTIAL_CLASSES:
            assert issubclass(C, Credential), "%s must be a subclass of Credential" % C

        # First credential should bear the name and url
        credentials = [self._CREDENTIAL_CLASSES[0](self.name, url=self.url, keyring=self._keyring)]
        # and we just reuse its _FIELDS for _ask_field_value etc
        self._FIELDS = credentials[0]._FIELDS
        # the rest with index suffix, but storing themselves in the same keyring
        for iC, C in enumerate(self._CREDENTIAL_CLASSES[1:]):
            credentials.append(
                C(name="%s:%d" % (self.name, iC + 1), url=None, keyring=self._keyring)
            )
        self._credentials = credentials

        super(CompositeCredential, self)._prepare()

    # Here it becomes tricky, since theoretically it is the "tail"
    # ones which might expire etc, so we wouldn't exactly know what
    # new credentials outside process wanted -- it would be silly to ask
    # right away the "entry" credentials if it is just the expiration of the
    # tail credentials
    def enter_new(self):
        # should invalidate/remove all tail credentials to avoid failing attempts to login
        self._credentials[0].enter_new()
        for c in self._credentials[1:]:
            c.delete()

    def __call__(self):
        """Obtain credentials from a keyring and if any is not known -- ask"""
        # Start from the tail until we have credentials set
        idx = len(self._credentials) - 1
        for c in self._credentials[::-1]:
            if c.is_known and not c.is_expired:
                break
            idx -= 1

        if idx < 0:
            # none was known, all the same -- start with the first one
            idx = 0

        # TODO: consider moving logic of traversal into authenticator since it is
        # the one spitting out authentication error etc
        # Theoretically we could just reuse 'fields' from adapter in the next step
        # but let's do full circle, so that if any "normalization" is done by
        # Credentials we take that into account
        for c, adapter, next_c in zip(
                self._credentials[idx:],
                self._CREDENTIAL_ADAPTERS[idx:],
                self._credentials[idx + 1:]):
            fields = c()
            next_fields = adapter(self, **fields)
            next_c.set(**next_fields)

        return self._credentials[-1]()


def _nda_adapter(composite, user=None, password=None):
    from datalad.support.third.nda_aws_token_generator import NDATokenGenerator
    gen = NDATokenGenerator()
    token = gen.generate_token(user, password)
    # There are also session and expiration we ignore... TODO anything about it?!!!
    # we could create a derived AWS_S3 which would also store session and expiration
    # and then may be Composite could use those????
    return dict(key_id=token.access_key, secret_id=token.secret_key,
                session=token.session, expiration=token.expiration)


class NDA_S3(CompositeCredential):
    """Credential to access NDA AWS

    So for NDA we need a credential which is a composite credential.
    User provides UserPassword and then some adapter generates AWS_S3
    out of it
    """
    _CREDENTIAL_CLASSES = (UserPassword, AWS_S3)
    _CREDENTIAL_ADAPTERS = (_nda_adapter,)


def _loris_adapter(composite, user=None, password=None, **kwargs):
    from datalad.support.third.loris_token_generator import LORISTokenGenerator

    gen = LORISTokenGenerator(url=composite.url)
    token = gen.generate_token(user, password)

    return dict(token=token)


class LORIS_Token(CompositeCredential):
    _CREDENTIAL_CLASSES = (UserPassword, Token)
    _CREDENTIAL_ADAPTERS = (_loris_adapter,)

    def __init__(self, name, url=None, keyring=None):
        super(CompositeCredential, self).__init__(name, url, keyring)


CREDENTIAL_TYPES = {
    'user_password': UserPassword,
    'aws-s3': AWS_S3,
    'nda-s3': NDA_S3,
    'token': Token,
    'loris-token': LORIS_Token
}

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user