We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[IP-Adapter] add a 'unload_ip_adapter' method
fix https://github.com/huggingface/diffusers/issues/6088

Is anyone interested in adding a `unload_ip_adapter` method? it needs to undo everything we did with `load_ip_adapter` method:
https://github.com/huggingface/diffusers/blob/93ea26f272f69a0e27afaebc96b68a2221a7eda0/src/diffusers/loaders/ip_adapter.py#L46

It needs to do below 3 things:
1. remove `image_encoder`
2. undo the changes we made to the UNet structure here (https://github.com/huggingface/diffusers/blob/93ea26f272f69a0e27afaebc96b68a2221a7eda0/src/diffusers/loaders/unet.py#L82), i.e. 
        ```python
        self.encoder_hid_proj = None
        self.config.encoder_hid_dim_type = None
       ```
3. reset the attention processor 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/diffusers/loaders/ip_adapter.py`
Content:
```
1 # Copyright 2023 The HuggingFace Team. All rights reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 import os
15 from typing import Dict, Union
16 
17 import torch
18 from huggingface_hub.utils import validate_hf_hub_args
19 from safetensors import safe_open
20 
21 from ..utils import (
22     _get_model_file,
23     is_transformers_available,
24     logging,
25 )
26 
27 
28 if is_transformers_available():
29     from transformers import (
30         CLIPImageProcessor,
31         CLIPVisionModelWithProjection,
32     )
33 
34     from ..models.attention_processor import (
35         IPAdapterAttnProcessor,
36         IPAdapterAttnProcessor2_0,
37     )
38 
39 logger = logging.get_logger(__name__)
40 
41 
42 class IPAdapterMixin:
43     """Mixin for handling IP Adapters."""
44 
45     @validate_hf_hub_args
46     def load_ip_adapter(
47         self,
48         pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]],
49         subfolder: str,
50         weight_name: str,
51         **kwargs,
52     ):
53         """
54         Parameters:
55             pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`):
56                 Can be either:
57 
58                     - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on
59                       the Hub.
60                     - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved
61                       with [`ModelMixin.save_pretrained`].
62                     - A [torch state
63                       dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict).
64 
65             cache_dir (`Union[str, os.PathLike]`, *optional*):
66                 Path to a directory where a downloaded pretrained model configuration is cached if the standard cache
67                 is not used.
68             force_download (`bool`, *optional*, defaults to `False`):
69                 Whether or not to force the (re-)download of the model weights and configuration files, overriding the
70                 cached versions if they exist.
71             resume_download (`bool`, *optional*, defaults to `False`):
72                 Whether or not to resume downloading the model weights and configuration files. If set to `False`, any
73                 incompletely downloaded files are deleted.
74             proxies (`Dict[str, str]`, *optional*):
75                 A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',
76                 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
77             local_files_only (`bool`, *optional*, defaults to `False`):
78                 Whether to only load local model weights and configuration files or not. If set to `True`, the model
79                 won't be downloaded from the Hub.
80             token (`str` or *bool*, *optional*):
81                 The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from
82                 `diffusers-cli login` (stored in `~/.huggingface`) is used.
83             revision (`str`, *optional*, defaults to `"main"`):
84                 The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier
85                 allowed by Git.
86             subfolder (`str`, *optional*, defaults to `""`):
87                 The subfolder location of a model file within a larger model repository on the Hub or locally.
88         """
89 
90         # Load the main state dict first.
91         cache_dir = kwargs.pop("cache_dir", None)
92         force_download = kwargs.pop("force_download", False)
93         resume_download = kwargs.pop("resume_download", False)
94         proxies = kwargs.pop("proxies", None)
95         local_files_only = kwargs.pop("local_files_only", None)
96         token = kwargs.pop("token", None)
97         revision = kwargs.pop("revision", None)
98 
99         user_agent = {
100             "file_type": "attn_procs_weights",
101             "framework": "pytorch",
102         }
103 
104         if not isinstance(pretrained_model_name_or_path_or_dict, dict):
105             model_file = _get_model_file(
106                 pretrained_model_name_or_path_or_dict,
107                 weights_name=weight_name,
108                 cache_dir=cache_dir,
109                 force_download=force_download,
110                 resume_download=resume_download,
111                 proxies=proxies,
112                 local_files_only=local_files_only,
113                 token=token,
114                 revision=revision,
115                 subfolder=subfolder,
116                 user_agent=user_agent,
117             )
118             if weight_name.endswith(".safetensors"):
119                 state_dict = {"image_proj": {}, "ip_adapter": {}}
120                 with safe_open(model_file, framework="pt", device="cpu") as f:
121                     for key in f.keys():
122                         if key.startswith("image_proj."):
123                             state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key)
124                         elif key.startswith("ip_adapter."):
125                             state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key)
126             else:
127                 state_dict = torch.load(model_file, map_location="cpu")
128         else:
129             state_dict = pretrained_model_name_or_path_or_dict
130 
131         keys = list(state_dict.keys())
132         if keys != ["image_proj", "ip_adapter"]:
133             raise ValueError("Required keys are (`image_proj` and `ip_adapter`) missing from the state dict.")
134 
135         # load CLIP image encoer here if it has not been registered to the pipeline yet
136         if hasattr(self, "image_encoder") and getattr(self, "image_encoder", None) is None:
137             if not isinstance(pretrained_model_name_or_path_or_dict, dict):
138                 logger.info(f"loading image_encoder from {pretrained_model_name_or_path_or_dict}")
139                 image_encoder = CLIPVisionModelWithProjection.from_pretrained(
140                     pretrained_model_name_or_path_or_dict,
141                     subfolder=os.path.join(subfolder, "image_encoder"),
142                 ).to(self.device, dtype=self.dtype)
143                 self.image_encoder = image_encoder
144             else:
145                 raise ValueError("`image_encoder` cannot be None when using IP Adapters.")
146 
147         # create feature extractor if it has not been registered to the pipeline yet
148         if hasattr(self, "feature_extractor") and getattr(self, "feature_extractor", None) is None:
149             self.feature_extractor = CLIPImageProcessor()
150 
151         # load ip-adapter into unet
152         self.unet._load_ip_adapter_weights(state_dict)
153 
154     def set_ip_adapter_scale(self, scale):
155         for attn_processor in self.unet.attn_processors.values():
156             if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)):
157                 attn_processor.scale = scale
158
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/diffusers/loaders/ip_adapter.py b/src/diffusers/loaders/ip_adapter.py
--- a/src/diffusers/loaders/ip_adapter.py
+++ b/src/diffusers/loaders/ip_adapter.py
@@ -132,7 +132,7 @@
         if keys != ["image_proj", "ip_adapter"]:
             raise ValueError("Required keys are (`image_proj` and `ip_adapter`) missing from the state dict.")
 
-        # load CLIP image encoer here if it has not been registered to the pipeline yet
+        # load CLIP image encoder here if it has not been registered to the pipeline yet
         if hasattr(self, "image_encoder") and getattr(self, "image_encoder", None) is None:
             if not isinstance(pretrained_model_name_or_path_or_dict, dict):
                 logger.info(f"loading image_encoder from {pretrained_model_name_or_path_or_dict}")
@@ -141,12 +141,14 @@
                     subfolder=os.path.join(subfolder, "image_encoder"),
                 ).to(self.device, dtype=self.dtype)
                 self.image_encoder = image_encoder
+                self.register_to_config(image_encoder=["transformers", "CLIPVisionModelWithProjection"])
             else:
                 raise ValueError("`image_encoder` cannot be None when using IP Adapters.")
 
         # create feature extractor if it has not been registered to the pipeline yet
         if hasattr(self, "feature_extractor") and getattr(self, "feature_extractor", None) is None:
             self.feature_extractor = CLIPImageProcessor()
+            self.register_to_config(feature_extractor=["transformers", "CLIPImageProcessor"])
 
         # load ip-adapter into unet
         self.unet._load_ip_adapter_weights(state_dict)
@@ -155,3 +157,32 @@
         for attn_processor in self.unet.attn_processors.values():
             if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)):
                 attn_processor.scale = scale
+
+    def unload_ip_adapter(self):
+        """
+        Unloads the IP Adapter weights
+
+        Examples:
+
+        ```python
+        >>> # Assuming `pipeline` is already loaded with the IP Adapter weights.
+        >>> pipeline.unload_ip_adapter()
+        >>> ...
+        ```
+        """
+        # remove CLIP image encoder
+        if hasattr(self, "image_encoder") and getattr(self, "image_encoder", None) is not None:
+            self.image_encoder = None
+            self.register_to_config(image_encoder=[None, None])
+
+        # remove feature extractor
+        if hasattr(self, "feature_extractor") and getattr(self, "feature_extractor", None) is not None:
+            self.feature_extractor = None
+            self.register_to_config(feature_extractor=[None, None])
+
+        # remove hidden encoder
+        self.unet.encoder_hid_proj = None
+        self.config.encoder_hid_dim_type = None
+
+        # restore original Unet attention processors layers
+        self.unet.set_default_attn_processor()

{"golden_diff": "diff --git a/src/diffusers/loaders/ip_adapter.py b/src/diffusers/loaders/ip_adapter.py\n--- a/src/diffusers/loaders/ip_adapter.py\n+++ b/src/diffusers/loaders/ip_adapter.py\n@@ -132,7 +132,7 @@\n         if keys != [\"image_proj\", \"ip_adapter\"]:\n             raise ValueError(\"Required keys are (`image_proj` and `ip_adapter`) missing from the state dict.\")\n \n-        # load CLIP image encoer here if it has not been registered to the pipeline yet\n+        # load CLIP image encoder here if it has not been registered to the pipeline yet\n         if hasattr(self, \"image_encoder\") and getattr(self, \"image_encoder\", None) is None:\n             if not isinstance(pretrained_model_name_or_path_or_dict, dict):\n                 logger.info(f\"loading image_encoder from {pretrained_model_name_or_path_or_dict}\")\n@@ -141,12 +141,14 @@\n                     subfolder=os.path.join(subfolder, \"image_encoder\"),\n                 ).to(self.device, dtype=self.dtype)\n                 self.image_encoder = image_encoder\n+                self.register_to_config(image_encoder=[\"transformers\", \"CLIPVisionModelWithProjection\"])\n             else:\n                 raise ValueError(\"`image_encoder` cannot be None when using IP Adapters.\")\n \n         # create feature extractor if it has not been registered to the pipeline yet\n         if hasattr(self, \"feature_extractor\") and getattr(self, \"feature_extractor\", None) is None:\n             self.feature_extractor = CLIPImageProcessor()\n+            self.register_to_config(feature_extractor=[\"transformers\", \"CLIPImageProcessor\"])\n \n         # load ip-adapter into unet\n         self.unet._load_ip_adapter_weights(state_dict)\n@@ -155,3 +157,32 @@\n         for attn_processor in self.unet.attn_processors.values():\n             if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)):\n                 attn_processor.scale = scale\n+\n+    def unload_ip_adapter(self):\n+        \"\"\"\n+        Unloads the IP Adapter weights\n+\n+        Examples:\n+\n+        ```python\n+        >>> # Assuming `pipeline` is already loaded with the IP Adapter weights.\n+        >>> pipeline.unload_ip_adapter()\n+        >>> ...\n+        ```\n+        \"\"\"\n+        # remove CLIP image encoder\n+        if hasattr(self, \"image_encoder\") and getattr(self, \"image_encoder\", None) is not None:\n+            self.image_encoder = None\n+            self.register_to_config(image_encoder=[None, None])\n+\n+        # remove feature extractor\n+        if hasattr(self, \"feature_extractor\") and getattr(self, \"feature_extractor\", None) is not None:\n+            self.feature_extractor = None\n+            self.register_to_config(feature_extractor=[None, None])\n+\n+        # remove hidden encoder\n+        self.unet.encoder_hid_proj = None\n+        self.config.encoder_hid_dim_type = None\n+\n+        # restore original Unet attention processors layers\n+        self.unet.set_default_attn_processor()\n", "issue": "[IP-Adapter] add a 'unload_ip_adapter' method\nfix https://github.com/huggingface/diffusers/issues/6088\r\n\r\nIs anyone interested in adding a `unload_ip_adapter` method? it needs to undo everything we did with `load_ip_adapter` method:\r\nhttps://github.com/huggingface/diffusers/blob/93ea26f272f69a0e27afaebc96b68a2221a7eda0/src/diffusers/loaders/ip_adapter.py#L46\r\n\r\nIt needs to do below 3 things:\r\n1. remove `image_encoder`\r\n2. undo the changes we made to the UNet structure here (https://github.com/huggingface/diffusers/blob/93ea26f272f69a0e27afaebc96b68a2221a7eda0/src/diffusers/loaders/unet.py#L82), i.e. \r\n        ```python\r\n        self.encoder_hid_proj = None\r\n        self.config.encoder_hid_dim_type = None\r\n       ```\r\n3. reset the attention processor \n", "before_files": [{"content": "# Copyright 2023 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport os\nfrom typing import Dict, Union\n\nimport torch\nfrom huggingface_hub.utils import validate_hf_hub_args\nfrom safetensors import safe_open\n\nfrom ..utils import (\n    _get_model_file,\n    is_transformers_available,\n    logging,\n)\n\n\nif is_transformers_available():\n    from transformers import (\n        CLIPImageProcessor,\n        CLIPVisionModelWithProjection,\n    )\n\n    from ..models.attention_processor import (\n        IPAdapterAttnProcessor,\n        IPAdapterAttnProcessor2_0,\n    )\n\nlogger = logging.get_logger(__name__)\n\n\nclass IPAdapterMixin:\n    \"\"\"Mixin for handling IP Adapters.\"\"\"\n\n    @validate_hf_hub_args\n    def load_ip_adapter(\n        self,\n        pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]],\n        subfolder: str,\n        weight_name: str,\n        **kwargs,\n    ):\n        \"\"\"\n        Parameters:\n            pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`):\n                Can be either:\n\n                    - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on\n                      the Hub.\n                    - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved\n                      with [`ModelMixin.save_pretrained`].\n                    - A [torch state\n                      dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict).\n\n            cache_dir (`Union[str, os.PathLike]`, *optional*):\n                Path to a directory where a downloaded pretrained model configuration is cached if the standard cache\n                is not used.\n            force_download (`bool`, *optional*, defaults to `False`):\n                Whether or not to force the (re-)download of the model weights and configuration files, overriding the\n                cached versions if they exist.\n            resume_download (`bool`, *optional*, defaults to `False`):\n                Whether or not to resume downloading the model weights and configuration files. If set to `False`, any\n                incompletely downloaded files are deleted.\n            proxies (`Dict[str, str]`, *optional*):\n                A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',\n                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.\n            local_files_only (`bool`, *optional*, defaults to `False`):\n                Whether to only load local model weights and configuration files or not. If set to `True`, the model\n                won't be downloaded from the Hub.\n            token (`str` or *bool*, *optional*):\n                The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from\n                `diffusers-cli login` (stored in `~/.huggingface`) is used.\n            revision (`str`, *optional*, defaults to `\"main\"`):\n                The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier\n                allowed by Git.\n            subfolder (`str`, *optional*, defaults to `\"\"`):\n                The subfolder location of a model file within a larger model repository on the Hub or locally.\n        \"\"\"\n\n        # Load the main state dict first.\n        cache_dir = kwargs.pop(\"cache_dir\", None)\n        force_download = kwargs.pop(\"force_download\", False)\n        resume_download = kwargs.pop(\"resume_download\", False)\n        proxies = kwargs.pop(\"proxies\", None)\n        local_files_only = kwargs.pop(\"local_files_only\", None)\n        token = kwargs.pop(\"token\", None)\n        revision = kwargs.pop(\"revision\", None)\n\n        user_agent = {\n            \"file_type\": \"attn_procs_weights\",\n            \"framework\": \"pytorch\",\n        }\n\n        if not isinstance(pretrained_model_name_or_path_or_dict, dict):\n            model_file = _get_model_file(\n                pretrained_model_name_or_path_or_dict,\n                weights_name=weight_name,\n                cache_dir=cache_dir,\n                force_download=force_download,\n                resume_download=resume_download,\n                proxies=proxies,\n                local_files_only=local_files_only,\n                token=token,\n                revision=revision,\n                subfolder=subfolder,\n                user_agent=user_agent,\n            )\n            if weight_name.endswith(\".safetensors\"):\n                state_dict = {\"image_proj\": {}, \"ip_adapter\": {}}\n                with safe_open(model_file, framework=\"pt\", device=\"cpu\") as f:\n                    for key in f.keys():\n                        if key.startswith(\"image_proj.\"):\n                            state_dict[\"image_proj\"][key.replace(\"image_proj.\", \"\")] = f.get_tensor(key)\n                        elif key.startswith(\"ip_adapter.\"):\n                            state_dict[\"ip_adapter\"][key.replace(\"ip_adapter.\", \"\")] = f.get_tensor(key)\n            else:\n                state_dict = torch.load(model_file, map_location=\"cpu\")\n        else:\n            state_dict = pretrained_model_name_or_path_or_dict\n\n        keys = list(state_dict.keys())\n        if keys != [\"image_proj\", \"ip_adapter\"]:\n            raise ValueError(\"Required keys are (`image_proj` and `ip_adapter`) missing from the state dict.\")\n\n        # load CLIP image encoer here if it has not been registered to the pipeline yet\n        if hasattr(self, \"image_encoder\") and getattr(self, \"image_encoder\", None) is None:\n            if not isinstance(pretrained_model_name_or_path_or_dict, dict):\n                logger.info(f\"loading image_encoder from {pretrained_model_name_or_path_or_dict}\")\n                image_encoder = CLIPVisionModelWithProjection.from_pretrained(\n                    pretrained_model_name_or_path_or_dict,\n                    subfolder=os.path.join(subfolder, \"image_encoder\"),\n                ).to(self.device, dtype=self.dtype)\n                self.image_encoder = image_encoder\n            else:\n                raise ValueError(\"`image_encoder` cannot be None when using IP Adapters.\")\n\n        # create feature extractor if it has not been registered to the pipeline yet\n        if hasattr(self, \"feature_extractor\") and getattr(self, \"feature_extractor\", None) is None:\n            self.feature_extractor = CLIPImageProcessor()\n\n        # load ip-adapter into unet\n        self.unet._load_ip_adapter_weights(state_dict)\n\n    def set_ip_adapter_scale(self, scale):\n        for attn_processor in self.unet.attn_processors.values():\n            if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)):\n                attn_processor.scale = scale\n", "path": "src/diffusers/loaders/ip_adapter.py"}], "after_files": [{"content": "# Copyright 2023 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport os\nfrom typing import Dict, Union\n\nimport torch\nfrom huggingface_hub.utils import validate_hf_hub_args\nfrom safetensors import safe_open\n\nfrom ..utils import (\n    _get_model_file,\n    is_transformers_available,\n    logging,\n)\n\n\nif is_transformers_available():\n    from transformers import (\n        CLIPImageProcessor,\n        CLIPVisionModelWithProjection,\n    )\n\n    from ..models.attention_processor import (\n        IPAdapterAttnProcessor,\n        IPAdapterAttnProcessor2_0,\n    )\n\nlogger = logging.get_logger(__name__)\n\n\nclass IPAdapterMixin:\n    \"\"\"Mixin for handling IP Adapters.\"\"\"\n\n    @validate_hf_hub_args\n    def load_ip_adapter(\n        self,\n        pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]],\n        subfolder: str,\n        weight_name: str,\n        **kwargs,\n    ):\n        \"\"\"\n        Parameters:\n            pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`):\n                Can be either:\n\n                    - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on\n                      the Hub.\n                    - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved\n                      with [`ModelMixin.save_pretrained`].\n                    - A [torch state\n                      dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict).\n\n            cache_dir (`Union[str, os.PathLike]`, *optional*):\n                Path to a directory where a downloaded pretrained model configuration is cached if the standard cache\n                is not used.\n            force_download (`bool`, *optional*, defaults to `False`):\n                Whether or not to force the (re-)download of the model weights and configuration files, overriding the\n                cached versions if they exist.\n            resume_download (`bool`, *optional*, defaults to `False`):\n                Whether or not to resume downloading the model weights and configuration files. If set to `False`, any\n                incompletely downloaded files are deleted.\n            proxies (`Dict[str, str]`, *optional*):\n                A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128',\n                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.\n            local_files_only (`bool`, *optional*, defaults to `False`):\n                Whether to only load local model weights and configuration files or not. If set to `True`, the model\n                won't be downloaded from the Hub.\n            token (`str` or *bool*, *optional*):\n                The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from\n                `diffusers-cli login` (stored in `~/.huggingface`) is used.\n            revision (`str`, *optional*, defaults to `\"main\"`):\n                The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier\n                allowed by Git.\n            subfolder (`str`, *optional*, defaults to `\"\"`):\n                The subfolder location of a model file within a larger model repository on the Hub or locally.\n        \"\"\"\n\n        # Load the main state dict first.\n        cache_dir = kwargs.pop(\"cache_dir\", None)\n        force_download = kwargs.pop(\"force_download\", False)\n        resume_download = kwargs.pop(\"resume_download\", False)\n        proxies = kwargs.pop(\"proxies\", None)\n        local_files_only = kwargs.pop(\"local_files_only\", None)\n        token = kwargs.pop(\"token\", None)\n        revision = kwargs.pop(\"revision\", None)\n\n        user_agent = {\n            \"file_type\": \"attn_procs_weights\",\n            \"framework\": \"pytorch\",\n        }\n\n        if not isinstance(pretrained_model_name_or_path_or_dict, dict):\n            model_file = _get_model_file(\n                pretrained_model_name_or_path_or_dict,\n                weights_name=weight_name,\n                cache_dir=cache_dir,\n                force_download=force_download,\n                resume_download=resume_download,\n                proxies=proxies,\n                local_files_only=local_files_only,\n                token=token,\n                revision=revision,\n                subfolder=subfolder,\n                user_agent=user_agent,\n            )\n            if weight_name.endswith(\".safetensors\"):\n                state_dict = {\"image_proj\": {}, \"ip_adapter\": {}}\n                with safe_open(model_file, framework=\"pt\", device=\"cpu\") as f:\n                    for key in f.keys():\n                        if key.startswith(\"image_proj.\"):\n                            state_dict[\"image_proj\"][key.replace(\"image_proj.\", \"\")] = f.get_tensor(key)\n                        elif key.startswith(\"ip_adapter.\"):\n                            state_dict[\"ip_adapter\"][key.replace(\"ip_adapter.\", \"\")] = f.get_tensor(key)\n            else:\n                state_dict = torch.load(model_file, map_location=\"cpu\")\n        else:\n            state_dict = pretrained_model_name_or_path_or_dict\n\n        keys = list(state_dict.keys())\n        if keys != [\"image_proj\", \"ip_adapter\"]:\n            raise ValueError(\"Required keys are (`image_proj` and `ip_adapter`) missing from the state dict.\")\n\n        # load CLIP image encoder here if it has not been registered to the pipeline yet\n        if hasattr(self, \"image_encoder\") and getattr(self, \"image_encoder\", None) is None:\n            if not isinstance(pretrained_model_name_or_path_or_dict, dict):\n                logger.info(f\"loading image_encoder from {pretrained_model_name_or_path_or_dict}\")\n                image_encoder = CLIPVisionModelWithProjection.from_pretrained(\n                    pretrained_model_name_or_path_or_dict,\n                    subfolder=os.path.join(subfolder, \"image_encoder\"),\n                ).to(self.device, dtype=self.dtype)\n                self.image_encoder = image_encoder\n                self.register_to_config(image_encoder=[\"transformers\", \"CLIPVisionModelWithProjection\"])\n            else:\n                raise ValueError(\"`image_encoder` cannot be None when using IP Adapters.\")\n\n        # create feature extractor if it has not been registered to the pipeline yet\n        if hasattr(self, \"feature_extractor\") and getattr(self, \"feature_extractor\", None) is None:\n            self.feature_extractor = CLIPImageProcessor()\n            self.register_to_config(feature_extractor=[\"transformers\", \"CLIPImageProcessor\"])\n\n        # load ip-adapter into unet\n        self.unet._load_ip_adapter_weights(state_dict)\n\n    def set_ip_adapter_scale(self, scale):\n        for attn_processor in self.unet.attn_processors.values():\n            if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)):\n                attn_processor.scale = scale\n\n    def unload_ip_adapter(self):\n        \"\"\"\n        Unloads the IP Adapter weights\n\n        Examples:\n\n        ```python\n        >>> # Assuming `pipeline` is already loaded with the IP Adapter weights.\n        >>> pipeline.unload_ip_adapter()\n        >>> ...\n        ```\n        \"\"\"\n        # remove CLIP image encoder\n        if hasattr(self, \"image_encoder\") and getattr(self, \"image_encoder\", None) is not None:\n            self.image_encoder = None\n            self.register_to_config(image_encoder=[None, None])\n\n        # remove feature extractor\n        if hasattr(self, \"feature_extractor\") and getattr(self, \"feature_extractor\", None) is not None:\n            self.feature_extractor = None\n            self.register_to_config(feature_extractor=[None, None])\n\n        # remove hidden encoder\n        self.unet.encoder_hid_proj = None\n        self.config.encoder_hid_dim_type = None\n\n        # restore original Unet attention processors layers\n        self.unet.set_default_attn_processor()\n", "path": "src/diffusers/loaders/ip_adapter.py"}]}