We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Unable to load Automatic1111 LoRa due rank being higher than "min(in_features, out_features)}"
### Describe the bug

Hi! I'm unable to load a LoRa as I get the following error.

`ValueError: LoRA rank 384 must be less or equal than 320`

The LoRa works perfectly if the ValueError check is removed from the code. Is it possible to override this error? I don't understand why it must be less or equal. Thanks!

### Reproduction

Run the interference script with [this](https://civitai.com/models/7227/urushi) LoRa
Get error about the LoRa rank

### Logs

```
  File "/home/wizard/pipelines/utils.py", line 465, in load_lora_weights_diffusers
    loader.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=unet)
  File "/home/wizard/lib/python3.9/site-packages/diffusers/loaders.py", line 1233, in load_lora_into_unet
    unet.load_attn_procs(state_dict, network_alphas=network_alphas)
  File "/home/wizardlib/python3.9/site-packages/diffusers/loaders.py", line 376, in load_attn_procs
    lora = LoRAConv2dLayer(
  File "/home/wizard/lib/python3.9/site-packages/diffusers/models/lora.py", line 58, in __init__
    raise ValueError(f"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}")
```


### System Info

- `diffusers` version: 0.19.3
- Platform: Linux-5.15.107+-x86_64-with-glibc2.31
- Python version: 3.9.16
- PyTorch version (GPU?): 2.0.1+cu118 (True)
- Huggingface_hub version: 0.16.4
- Transformers version: 4.29.2
- Accelerate version: 0.16.0
- xFormers version: 0.0.20
- Using GPU in script?: yes
- Using distributed or parallel set-up in script?: no

### Who can help?

@williamberman @patrickvonplaten @saya

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/diffusers/models/lora.py`
Content:
```
1 # Copyright 2023 The HuggingFace Team. All rights reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 from typing import Optional
16 
17 import torch.nn.functional as F
18 from torch import nn
19 
20 
21 class LoRALinearLayer(nn.Module):
22     def __init__(self, in_features, out_features, rank=4, network_alpha=None, device=None, dtype=None):
23         super().__init__()
24 
25         if rank > min(in_features, out_features):
26             raise ValueError(f"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}")
27 
28         self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype)
29         self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype)
30         # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.
31         # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning
32         self.network_alpha = network_alpha
33         self.rank = rank
34 
35         nn.init.normal_(self.down.weight, std=1 / rank)
36         nn.init.zeros_(self.up.weight)
37 
38     def forward(self, hidden_states):
39         orig_dtype = hidden_states.dtype
40         dtype = self.down.weight.dtype
41 
42         down_hidden_states = self.down(hidden_states.to(dtype))
43         up_hidden_states = self.up(down_hidden_states)
44 
45         if self.network_alpha is not None:
46             up_hidden_states *= self.network_alpha / self.rank
47 
48         return up_hidden_states.to(orig_dtype)
49 
50 
51 class LoRAConv2dLayer(nn.Module):
52     def __init__(
53         self, in_features, out_features, rank=4, kernel_size=(1, 1), stride=(1, 1), padding=0, network_alpha=None
54     ):
55         super().__init__()
56 
57         if rank > min(in_features, out_features):
58             raise ValueError(f"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}")
59 
60         self.down = nn.Conv2d(in_features, rank, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)
61         # according to the official kohya_ss trainer kernel_size are always fixed for the up layer
62         # # see: https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L129
63         self.up = nn.Conv2d(rank, out_features, kernel_size=(1, 1), stride=(1, 1), bias=False)
64 
65         # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.
66         # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning
67         self.network_alpha = network_alpha
68         self.rank = rank
69 
70         nn.init.normal_(self.down.weight, std=1 / rank)
71         nn.init.zeros_(self.up.weight)
72 
73     def forward(self, hidden_states):
74         orig_dtype = hidden_states.dtype
75         dtype = self.down.weight.dtype
76 
77         down_hidden_states = self.down(hidden_states.to(dtype))
78         up_hidden_states = self.up(down_hidden_states)
79 
80         if self.network_alpha is not None:
81             up_hidden_states *= self.network_alpha / self.rank
82 
83         return up_hidden_states.to(orig_dtype)
84 
85 
86 class LoRACompatibleConv(nn.Conv2d):
87     """
88     A convolutional layer that can be used with LoRA.
89     """
90 
91     def __init__(self, *args, lora_layer: Optional[LoRAConv2dLayer] = None, **kwargs):
92         super().__init__(*args, **kwargs)
93         self.lora_layer = lora_layer
94 
95     def set_lora_layer(self, lora_layer: Optional[LoRAConv2dLayer]):
96         self.lora_layer = lora_layer
97 
98     def forward(self, x):
99         if self.lora_layer is None:
100             # make sure to the functional Conv2D function as otherwise torch.compile's graph will break
101             # see: https://github.com/huggingface/diffusers/pull/4315
102             return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
103         else:
104             return super().forward(x) + self.lora_layer(x)
105 
106 
107 class LoRACompatibleLinear(nn.Linear):
108     """
109     A Linear layer that can be used with LoRA.
110     """
111 
112     def __init__(self, *args, lora_layer: Optional[LoRALinearLayer] = None, **kwargs):
113         super().__init__(*args, **kwargs)
114         self.lora_layer = lora_layer
115 
116     def set_lora_layer(self, lora_layer: Optional[LoRAConv2dLayer]):
117         self.lora_layer = lora_layer
118 
119     def forward(self, x):
120         if self.lora_layer is None:
121             return super().forward(x)
122         else:
123             return super().forward(x) + self.lora_layer(x)
124
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/diffusers/models/lora.py b/src/diffusers/models/lora.py
--- a/src/diffusers/models/lora.py
+++ b/src/diffusers/models/lora.py
@@ -22,9 +22,6 @@
     def __init__(self, in_features, out_features, rank=4, network_alpha=None, device=None, dtype=None):
         super().__init__()
 
-        if rank > min(in_features, out_features):
-            raise ValueError(f"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}")
-
         self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype)
         self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype)
         # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.
@@ -54,9 +51,6 @@
     ):
         super().__init__()
 
-        if rank > min(in_features, out_features):
-            raise ValueError(f"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}")
-
         self.down = nn.Conv2d(in_features, rank, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)
         # according to the official kohya_ss trainer kernel_size are always fixed for the up layer
         # # see: https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L129

{"golden_diff": "diff --git a/src/diffusers/models/lora.py b/src/diffusers/models/lora.py\n--- a/src/diffusers/models/lora.py\n+++ b/src/diffusers/models/lora.py\n@@ -22,9 +22,6 @@\n     def __init__(self, in_features, out_features, rank=4, network_alpha=None, device=None, dtype=None):\n         super().__init__()\n \n-        if rank > min(in_features, out_features):\n-            raise ValueError(f\"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}\")\n-\n         self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype)\n         self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype)\n         # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.\n@@ -54,9 +51,6 @@\n     ):\n         super().__init__()\n \n-        if rank > min(in_features, out_features):\n-            raise ValueError(f\"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}\")\n-\n         self.down = nn.Conv2d(in_features, rank, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)\n         # according to the official kohya_ss trainer kernel_size are always fixed for the up layer\n         # # see: https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L129\n", "issue": "Unable to load Automatic1111 LoRa due rank being higher than \"min(in_features, out_features)}\"\n### Describe the bug\r\n\r\nHi! I'm unable to load a LoRa as I get the following error.\r\n\r\n`ValueError: LoRA rank 384 must be less or equal than 320`\r\n\r\nThe LoRa works perfectly if the ValueError check is removed from the code. Is it possible to override this error? I don't understand why it must be less or equal. Thanks!\r\n\r\n### Reproduction\r\n\r\nRun the interference script with [this](https://civitai.com/models/7227/urushi) LoRa\r\nGet error about the LoRa rank\r\n\r\n### Logs\r\n\r\n```\r\n  File \"/home/wizard/pipelines/utils.py\", line 465, in load_lora_weights_diffusers\r\n    loader.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=unet)\r\n  File \"/home/wizard/lib/python3.9/site-packages/diffusers/loaders.py\", line 1233, in load_lora_into_unet\r\n    unet.load_attn_procs(state_dict, network_alphas=network_alphas)\r\n  File \"/home/wizardlib/python3.9/site-packages/diffusers/loaders.py\", line 376, in load_attn_procs\r\n    lora = LoRAConv2dLayer(\r\n  File \"/home/wizard/lib/python3.9/site-packages/diffusers/models/lora.py\", line 58, in __init__\r\n    raise ValueError(f\"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}\")\r\n```\r\n\r\n\r\n### System Info\r\n\r\n- `diffusers` version: 0.19.3\r\n- Platform: Linux-5.15.107+-x86_64-with-glibc2.31\r\n- Python version: 3.9.16\r\n- PyTorch version (GPU?): 2.0.1+cu118 (True)\r\n- Huggingface_hub version: 0.16.4\r\n- Transformers version: 4.29.2\r\n- Accelerate version: 0.16.0\r\n- xFormers version: 0.0.20\r\n- Using GPU in script?: yes\r\n- Using distributed or parallel set-up in script?: no\r\n\r\n### Who can help?\r\n\r\n@williamberman @patrickvonplaten @saya\n", "before_files": [{"content": "# Copyright 2023 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Optional\n\nimport torch.nn.functional as F\nfrom torch import nn\n\n\nclass LoRALinearLayer(nn.Module):\n    def __init__(self, in_features, out_features, rank=4, network_alpha=None, device=None, dtype=None):\n        super().__init__()\n\n        if rank > min(in_features, out_features):\n            raise ValueError(f\"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}\")\n\n        self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype)\n        self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype)\n        # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.\n        # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning\n        self.network_alpha = network_alpha\n        self.rank = rank\n\n        nn.init.normal_(self.down.weight, std=1 / rank)\n        nn.init.zeros_(self.up.weight)\n\n    def forward(self, hidden_states):\n        orig_dtype = hidden_states.dtype\n        dtype = self.down.weight.dtype\n\n        down_hidden_states = self.down(hidden_states.to(dtype))\n        up_hidden_states = self.up(down_hidden_states)\n\n        if self.network_alpha is not None:\n            up_hidden_states *= self.network_alpha / self.rank\n\n        return up_hidden_states.to(orig_dtype)\n\n\nclass LoRAConv2dLayer(nn.Module):\n    def __init__(\n        self, in_features, out_features, rank=4, kernel_size=(1, 1), stride=(1, 1), padding=0, network_alpha=None\n    ):\n        super().__init__()\n\n        if rank > min(in_features, out_features):\n            raise ValueError(f\"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}\")\n\n        self.down = nn.Conv2d(in_features, rank, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)\n        # according to the official kohya_ss trainer kernel_size are always fixed for the up layer\n        # # see: https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L129\n        self.up = nn.Conv2d(rank, out_features, kernel_size=(1, 1), stride=(1, 1), bias=False)\n\n        # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.\n        # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning\n        self.network_alpha = network_alpha\n        self.rank = rank\n\n        nn.init.normal_(self.down.weight, std=1 / rank)\n        nn.init.zeros_(self.up.weight)\n\n    def forward(self, hidden_states):\n        orig_dtype = hidden_states.dtype\n        dtype = self.down.weight.dtype\n\n        down_hidden_states = self.down(hidden_states.to(dtype))\n        up_hidden_states = self.up(down_hidden_states)\n\n        if self.network_alpha is not None:\n            up_hidden_states *= self.network_alpha / self.rank\n\n        return up_hidden_states.to(orig_dtype)\n\n\nclass LoRACompatibleConv(nn.Conv2d):\n    \"\"\"\n    A convolutional layer that can be used with LoRA.\n    \"\"\"\n\n    def __init__(self, *args, lora_layer: Optional[LoRAConv2dLayer] = None, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.lora_layer = lora_layer\n\n    def set_lora_layer(self, lora_layer: Optional[LoRAConv2dLayer]):\n        self.lora_layer = lora_layer\n\n    def forward(self, x):\n        if self.lora_layer is None:\n            # make sure to the functional Conv2D function as otherwise torch.compile's graph will break\n            # see: https://github.com/huggingface/diffusers/pull/4315\n            return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        else:\n            return super().forward(x) + self.lora_layer(x)\n\n\nclass LoRACompatibleLinear(nn.Linear):\n    \"\"\"\n    A Linear layer that can be used with LoRA.\n    \"\"\"\n\n    def __init__(self, *args, lora_layer: Optional[LoRALinearLayer] = None, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.lora_layer = lora_layer\n\n    def set_lora_layer(self, lora_layer: Optional[LoRAConv2dLayer]):\n        self.lora_layer = lora_layer\n\n    def forward(self, x):\n        if self.lora_layer is None:\n            return super().forward(x)\n        else:\n            return super().forward(x) + self.lora_layer(x)\n", "path": "src/diffusers/models/lora.py"}], "after_files": [{"content": "# Copyright 2023 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Optional\n\nimport torch.nn.functional as F\nfrom torch import nn\n\n\nclass LoRALinearLayer(nn.Module):\n    def __init__(self, in_features, out_features, rank=4, network_alpha=None, device=None, dtype=None):\n        super().__init__()\n\n        self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype)\n        self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype)\n        # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.\n        # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning\n        self.network_alpha = network_alpha\n        self.rank = rank\n\n        nn.init.normal_(self.down.weight, std=1 / rank)\n        nn.init.zeros_(self.up.weight)\n\n    def forward(self, hidden_states):\n        orig_dtype = hidden_states.dtype\n        dtype = self.down.weight.dtype\n\n        down_hidden_states = self.down(hidden_states.to(dtype))\n        up_hidden_states = self.up(down_hidden_states)\n\n        if self.network_alpha is not None:\n            up_hidden_states *= self.network_alpha / self.rank\n\n        return up_hidden_states.to(orig_dtype)\n\n\nclass LoRAConv2dLayer(nn.Module):\n    def __init__(\n        self, in_features, out_features, rank=4, kernel_size=(1, 1), stride=(1, 1), padding=0, network_alpha=None\n    ):\n        super().__init__()\n\n        self.down = nn.Conv2d(in_features, rank, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)\n        # according to the official kohya_ss trainer kernel_size are always fixed for the up layer\n        # # see: https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L129\n        self.up = nn.Conv2d(rank, out_features, kernel_size=(1, 1), stride=(1, 1), bias=False)\n\n        # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script.\n        # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning\n        self.network_alpha = network_alpha\n        self.rank = rank\n\n        nn.init.normal_(self.down.weight, std=1 / rank)\n        nn.init.zeros_(self.up.weight)\n\n    def forward(self, hidden_states):\n        orig_dtype = hidden_states.dtype\n        dtype = self.down.weight.dtype\n\n        down_hidden_states = self.down(hidden_states.to(dtype))\n        up_hidden_states = self.up(down_hidden_states)\n\n        if self.network_alpha is not None:\n            up_hidden_states *= self.network_alpha / self.rank\n\n        return up_hidden_states.to(orig_dtype)\n\n\nclass LoRACompatibleConv(nn.Conv2d):\n    \"\"\"\n    A convolutional layer that can be used with LoRA.\n    \"\"\"\n\n    def __init__(self, *args, lora_layer: Optional[LoRAConv2dLayer] = None, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.lora_layer = lora_layer\n\n    def set_lora_layer(self, lora_layer: Optional[LoRAConv2dLayer]):\n        self.lora_layer = lora_layer\n\n    def forward(self, x):\n        if self.lora_layer is None:\n            # make sure to the functional Conv2D function as otherwise torch.compile's graph will break\n            # see: https://github.com/huggingface/diffusers/pull/4315\n            return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        else:\n            return super().forward(x) + self.lora_layer(x)\n\n\nclass LoRACompatibleLinear(nn.Linear):\n    \"\"\"\n    A Linear layer that can be used with LoRA.\n    \"\"\"\n\n    def __init__(self, *args, lora_layer: Optional[LoRALinearLayer] = None, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.lora_layer = lora_layer\n\n    def set_lora_layer(self, lora_layer: Optional[LoRAConv2dLayer]):\n        self.lora_layer = lora_layer\n\n    def forward(self, x):\n        if self.lora_layer is None:\n            return super().forward(x)\n        else:\n            return super().forward(x) + self.lora_layer(x)\n", "path": "src/diffusers/models/lora.py"}]}