We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[New Model]: Phi-2 support for LoRA
### The model to consider.

Microsoft/Phi-2 with LoRA

### The closest model vllm already supports.

_No response_

### What's your difficulty of supporting the model you want?

_No response_

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/model_executor/models/phi.py`
Content:
```
1 # coding=utf-8
2 # Adapted from
3 # https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py
4 # Copyright 2023 The vLLM team.
5 # Copyright (c) Microsoft Corporation.
6 # Licensed under the MIT license.
7 #
8 # BSD 3-Clause License
9 #
10 # Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
11 # All rights reserved.
12 #
13 # Redistribution and use in source and binary forms, with or without
14 # modification, are permitted provided that the following conditions are met:
15 #
16 # * Redistributions of source code must retain the above copyright notice, this
17 #   list of conditions and the following disclaimer.
18 #
19 # * Redistributions in binary form must reproduce the above copyright notice,
20 #   this list of conditions and the following disclaimer in the documentation
21 #   and/or other materials provided with the distribution.
22 #
23 # * Neither the name of the copyright holder nor the names of its
24 #   contributors may be used to endorse or promote products derived from
25 #   this software without specific prior written permission.
26 #
27 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
28 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
30 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
31 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
33 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
34 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
35 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 """Inference-only Phi-1.5 model compatible with HuggingFace weights."""
38 from typing import Iterable, List, Optional, Tuple
39 
40 import torch
41 from torch import nn
42 from transformers import PretrainedConfig
43 
44 from vllm.attention import Attention, AttentionMetadata
45 from vllm.config import CacheConfig
46 from vllm.distributed import get_tensor_model_parallel_world_size
47 from vllm.model_executor.layers.activation import get_act_fn
48 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49                                                QKVParallelLinear,
50                                                RowParallelLinear)
51 from vllm.model_executor.layers.logits_processor import LogitsProcessor
52 from vllm.model_executor.layers.quantization.base_config import (
53     QuantizationConfig)
54 from vllm.model_executor.layers.rotary_embedding import get_rope
55 from vllm.model_executor.layers.sampler import Sampler
56 from vllm.model_executor.layers.vocab_parallel_embedding import (
57     ParallelLMHead, VocabParallelEmbedding)
58 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59 from vllm.model_executor.sampling_metadata import SamplingMetadata
60 from vllm.sequence import SamplerOutput
61 
62 
63 class PhiAttention(nn.Module):
64 
65     def __init__(self,
66                  config: PretrainedConfig,
67                  cache_config: Optional[CacheConfig] = None,
68                  quant_config: Optional[QuantizationConfig] = None):
69         super().__init__()
70         self.total_num_heads = config.num_attention_heads
71         self.hidden_size = config.hidden_size
72         self.head_size = self.hidden_size // self.total_num_heads
73 
74         tensor_model_parallel_world_size = (
75             get_tensor_model_parallel_world_size())
76         assert self.total_num_heads % tensor_model_parallel_world_size == 0
77         self.num_heads = (self.total_num_heads //
78                           tensor_model_parallel_world_size)
79 
80         # pylint: disable=C0103
81         self.qkv_proj = QKVParallelLinear(
82             self.hidden_size,
83             self.head_size,
84             self.total_num_heads,
85             bias=True,
86             quant_config=quant_config,
87         )
88         self.dense = RowParallelLinear(
89             self.hidden_size,
90             self.hidden_size,
91             quant_config=quant_config,
92         )
93 
94         scaling = self.head_size**-0.5
95         rotary_dim = int(config.partial_rotary_factor *
96                          (config.hidden_size // config.num_attention_heads))
97         assert rotary_dim % 2 == 0
98 
99         # pylint: disable=C0301
100         # Refer to:
101         # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518
102         rope_theta = 10000
103         max_position_embeddings = getattr(config, "n_positions", 2048)
104         self.rotary_emb = get_rope(
105             self.head_size,
106             rotary_dim=rotary_dim,
107             max_position=max_position_embeddings,
108             base=rope_theta,
109         )
110         self.attn = Attention(self.num_heads,
111                               self.head_size,
112                               scaling,
113                               cache_config=cache_config)
114 
115     def forward(
116         self,
117         position_ids: torch.Tensor,
118         hidden_states: torch.Tensor,
119         kv_cache: torch.Tensor,
120         attn_metadata: AttentionMetadata,
121     ) -> torch.Tensor:
122         qkv, _ = self.qkv_proj(hidden_states)
123         q, k, v = qkv.chunk(chunks=3, dim=-1)
124         q, k = self.rotary_emb(position_ids, q, k)
125         attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
126         output, _ = self.dense(attn_output)
127         return output
128 
129 
130 class PhiMLP(nn.Module):
131 
132     def __init__(self,
133                  config: PretrainedConfig,
134                  quant_config: Optional[QuantizationConfig] = None):
135         super().__init__()
136 
137         n_inner = getattr(config, "n_inner", None)
138         n_inner = n_inner if n_inner is not None else 4 * config.hidden_size
139 
140         self.fc1 = ColumnParallelLinear(
141             config.hidden_size,
142             n_inner,
143             quant_config=quant_config,
144         )
145         self.fc2 = RowParallelLinear(
146             n_inner,
147             config.hidden_size,
148             quant_config=quant_config,
149         )
150         self.act = get_act_fn(config.hidden_act, quant_config, n_inner)
151 
152     def forward(self, hidden_states):
153         hidden_states, _ = self.fc1(hidden_states)
154         hidden_states = self.act(hidden_states)
155         hidden_states, _ = self.fc2(hidden_states)
156         return hidden_states
157 
158 
159 class PhiLayer(nn.Module):
160 
161     def __init__(self,
162                  config: PretrainedConfig,
163                  cache_config: Optional[CacheConfig] = None,
164                  quant_config: Optional[QuantizationConfig] = None):
165         super().__init__()
166         self.input_layernorm = nn.LayerNorm(config.hidden_size,
167                                             eps=config.layer_norm_eps)
168         self.self_attn = PhiAttention(config, cache_config, quant_config)
169         self.mlp = PhiMLP(config, quant_config)
170 
171     def forward(
172         self,
173         position_ids: torch.Tensor,
174         hidden_states: torch.Tensor,
175         kv_cache: torch.Tensor,
176         attn_metadata: AttentionMetadata,
177     ) -> torch.Tensor:
178         residual = hidden_states
179         hidden_states = self.input_layernorm(hidden_states)
180         attn_outputs = self.self_attn(
181             position_ids=position_ids,
182             hidden_states=hidden_states,
183             kv_cache=kv_cache,
184             attn_metadata=attn_metadata,
185         )
186         feed_forward_hidden_states = self.mlp(hidden_states)
187         hidden_states = attn_outputs + feed_forward_hidden_states + residual
188         return hidden_states
189 
190 
191 class PhiModel(nn.Module):
192 
193     def __init__(self,
194                  config: PretrainedConfig,
195                  cache_config: Optional[CacheConfig] = None,
196                  quant_config: Optional[QuantizationConfig] = None):
197         super().__init__()
198         self.config = config
199         self.quant_config = quant_config
200         self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
201                                                    config.hidden_size)
202         self.layers = nn.ModuleList([
203             PhiLayer(config, cache_config, quant_config)
204             for _ in range(config.num_hidden_layers)
205         ])
206         self.final_layernorm = nn.LayerNorm(config.hidden_size,
207                                             eps=config.layer_norm_eps)
208 
209     def forward(
210         self,
211         input_ids: torch.Tensor,
212         positions: torch.Tensor,
213         kv_caches: List[torch.Tensor],
214         attn_metadata: AttentionMetadata,
215     ) -> torch.Tensor:
216         hidden_states = self.embed_tokens(input_ids)
217         for i in range(self.config.num_hidden_layers):
218             layer = self.layers[i]
219             hidden_states = layer(
220                 positions,
221                 hidden_states,
222                 kv_caches[i],
223                 attn_metadata,
224             )
225 
226         hidden_states = self.final_layernorm(hidden_states)
227 
228         return hidden_states
229 
230 
231 class PhiForCausalLM(nn.Module):
232 
233     def __init__(self,
234                  config: PretrainedConfig,
235                  cache_config: Optional[CacheConfig] = None,
236                  quant_config: Optional[QuantizationConfig] = None):
237         super().__init__()
238         self.config = config
239         self.quant_config = quant_config
240 
241         self.model = PhiModel(config, cache_config, quant_config)
242 
243         self.lm_head = ParallelLMHead(config.vocab_size,
244                                       config.hidden_size,
245                                       bias=True)
246         self.logits_processor = LogitsProcessor(config.vocab_size)
247         self.sampler = Sampler()
248 
249     def forward(
250         self,
251         input_ids: torch.Tensor,
252         positions: torch.Tensor,
253         kv_caches: List[torch.Tensor],
254         attn_metadata: AttentionMetadata,
255     ) -> torch.Tensor:
256         hidden_states = self.model(input_ids, positions, kv_caches,
257                                    attn_metadata)
258 
259         return hidden_states
260 
261     def compute_logits(self, hidden_states: torch.Tensor,
262                        sampling_metadata: SamplingMetadata) -> torch.Tensor:
263         logits = self.logits_processor(self.lm_head.weight, hidden_states,
264                                        sampling_metadata, self.lm_head.bias)
265         return logits
266 
267     def sample(
268         self,
269         logits: torch.Tensor,
270         sampling_metadata: SamplingMetadata,
271     ) -> Optional[SamplerOutput]:
272         next_tokens = self.sampler(logits, sampling_metadata)
273         return next_tokens
274 
275     def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
276         stacked_params_mapping = [
277             # (param_name, shard_name, shard_id)
278             ("qkv_proj", "q_proj", "q"),
279             ("qkv_proj", "k_proj", "k"),
280             ("qkv_proj", "v_proj", "v")
281         ]
282         params_dict = dict(self.named_parameters())
283 
284         for name, loaded_weight in weights:
285             if "rotary_emb.inv_freq" in name:
286                 continue
287 
288             for (param_name, weight_name, shard_id) in stacked_params_mapping:
289                 if weight_name not in name:
290                     continue
291                 name = name.replace(weight_name, param_name)
292                 # Skip loading extra bias for GPTQ models.
293                 if name.endswith(".bias") and name not in params_dict:
294                     continue
295                 param = params_dict[name]
296                 weight_loader = param.weight_loader
297                 weight_loader(param, loaded_weight, shard_id)
298                 break
299             else:
300                 # Skip loading extra bias for GPTQ models.
301                 if name.endswith(".bias") and name not in params_dict:
302                     continue
303                 # pylint: disable=E1136
304 
305                 param = params_dict[name]
306                 weight_loader = getattr(param, "weight_loader",
307                                         default_weight_loader)
308                 weight_loader(param, loaded_weight)
309
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/model_executor/models/phi.py b/vllm/model_executor/models/phi.py
--- a/vllm/model_executor/models/phi.py
+++ b/vllm/model_executor/models/phi.py
@@ -42,7 +42,7 @@
 from transformers import PretrainedConfig
 
 from vllm.attention import Attention, AttentionMetadata
-from vllm.config import CacheConfig
+from vllm.config import CacheConfig, LoRAConfig
 from vllm.distributed import get_tensor_model_parallel_world_size
 from vllm.model_executor.layers.activation import get_act_fn
 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -229,11 +229,32 @@
 
 
 class PhiForCausalLM(nn.Module):
-
-    def __init__(self,
-                 config: PretrainedConfig,
-                 cache_config: Optional[CacheConfig] = None,
-                 quant_config: Optional[QuantizationConfig] = None):
+    packed_modules_mapping = {
+        "qkv_proj": [
+            "q_proj",
+            "k_proj",
+            "v_proj",
+        ]
+    }
+
+    # LoRA specific attributes
+    supported_lora_modules = [
+        "qkv_proj",
+        "dense",
+        "fc1",
+        "fc2",
+    ]
+    embedding_modules = {}
+    embedding_padding_modules = []
+
+    def __init__(
+        self,
+        config: PretrainedConfig,
+        cache_config: Optional[CacheConfig] = None,
+        quant_config: Optional[QuantizationConfig] = None,
+        lora_config: Optional[LoRAConfig] = None,
+    ):
+        del lora_config  # Unused.
         super().__init__()
         self.config = config
         self.quant_config = quant_config

{"golden_diff": "diff --git a/vllm/model_executor/models/phi.py b/vllm/model_executor/models/phi.py\n--- a/vllm/model_executor/models/phi.py\n+++ b/vllm/model_executor/models/phi.py\n@@ -42,7 +42,7 @@\n from transformers import PretrainedConfig\n \n from vllm.attention import Attention, AttentionMetadata\n-from vllm.config import CacheConfig\n+from vllm.config import CacheConfig, LoRAConfig\n from vllm.distributed import get_tensor_model_parallel_world_size\n from vllm.model_executor.layers.activation import get_act_fn\n from vllm.model_executor.layers.linear import (ColumnParallelLinear,\n@@ -229,11 +229,32 @@\n \n \n class PhiForCausalLM(nn.Module):\n-\n-    def __init__(self,\n-                 config: PretrainedConfig,\n-                 cache_config: Optional[CacheConfig] = None,\n-                 quant_config: Optional[QuantizationConfig] = None):\n+    packed_modules_mapping = {\n+        \"qkv_proj\": [\n+            \"q_proj\",\n+            \"k_proj\",\n+            \"v_proj\",\n+        ]\n+    }\n+\n+    # LoRA specific attributes\n+    supported_lora_modules = [\n+        \"qkv_proj\",\n+        \"dense\",\n+        \"fc1\",\n+        \"fc2\",\n+    ]\n+    embedding_modules = {}\n+    embedding_padding_modules = []\n+\n+    def __init__(\n+        self,\n+        config: PretrainedConfig,\n+        cache_config: Optional[CacheConfig] = None,\n+        quant_config: Optional[QuantizationConfig] = None,\n+        lora_config: Optional[LoRAConfig] = None,\n+    ):\n+        del lora_config  # Unused.\n         super().__init__()\n         self.config = config\n         self.quant_config = quant_config\n", "issue": "[New Model]: Phi-2 support for LoRA\n### The model to consider.\n\nMicrosoft/Phi-2 with LoRA\n\n### The closest model vllm already supports.\n\n_No response_\n\n### What's your difficulty of supporting the model you want?\n\n_No response_\n", "before_files": [{"content": "# coding=utf-8\n# Adapted from\n# https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py\n# Copyright 2023 The vLLM team.\n# Copyright (c) Microsoft Corporation.\n# Licensed under the MIT license.\n#\n# BSD 3-Clause License\n#\n# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.\n# All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are met:\n#\n# * Redistributions of source code must retain the above copyright notice, this\n#   list of conditions and the following disclaimer.\n#\n# * Redistributions in binary form must reproduce the above copyright notice,\n#   this list of conditions and the following disclaimer in the documentation\n#   and/or other materials provided with the distribution.\n#\n# * Neither the name of the copyright holder nor the names of its\n#   contributors may be used to endorse or promote products derived from\n#   this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\n# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\"\"\"Inference-only Phi-1.5 model compatible with HuggingFace weights.\"\"\"\nfrom typing import Iterable, List, Optional, Tuple\n\nimport torch\nfrom torch import nn\nfrom transformers import PretrainedConfig\n\nfrom vllm.attention import Attention, AttentionMetadata\nfrom vllm.config import CacheConfig\nfrom vllm.distributed import get_tensor_model_parallel_world_size\nfrom vllm.model_executor.layers.activation import get_act_fn\nfrom vllm.model_executor.layers.linear import (ColumnParallelLinear,\n                                               QKVParallelLinear,\n                                               RowParallelLinear)\nfrom vllm.model_executor.layers.logits_processor import LogitsProcessor\nfrom vllm.model_executor.layers.quantization.base_config import (\n    QuantizationConfig)\nfrom vllm.model_executor.layers.rotary_embedding import get_rope\nfrom vllm.model_executor.layers.sampler import Sampler\nfrom vllm.model_executor.layers.vocab_parallel_embedding import (\n    ParallelLMHead, VocabParallelEmbedding)\nfrom vllm.model_executor.model_loader.weight_utils import default_weight_loader\nfrom vllm.model_executor.sampling_metadata import SamplingMetadata\nfrom vllm.sequence import SamplerOutput\n\n\nclass PhiAttention(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 cache_config: Optional[CacheConfig] = None,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n        self.total_num_heads = config.num_attention_heads\n        self.hidden_size = config.hidden_size\n        self.head_size = self.hidden_size // self.total_num_heads\n\n        tensor_model_parallel_world_size = (\n            get_tensor_model_parallel_world_size())\n        assert self.total_num_heads % tensor_model_parallel_world_size == 0\n        self.num_heads = (self.total_num_heads //\n                          tensor_model_parallel_world_size)\n\n        # pylint: disable=C0103\n        self.qkv_proj = QKVParallelLinear(\n            self.hidden_size,\n            self.head_size,\n            self.total_num_heads,\n            bias=True,\n            quant_config=quant_config,\n        )\n        self.dense = RowParallelLinear(\n            self.hidden_size,\n            self.hidden_size,\n            quant_config=quant_config,\n        )\n\n        scaling = self.head_size**-0.5\n        rotary_dim = int(config.partial_rotary_factor *\n                         (config.hidden_size // config.num_attention_heads))\n        assert rotary_dim % 2 == 0\n\n        # pylint: disable=C0301\n        # Refer to:\n        # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518\n        rope_theta = 10000\n        max_position_embeddings = getattr(config, \"n_positions\", 2048)\n        self.rotary_emb = get_rope(\n            self.head_size,\n            rotary_dim=rotary_dim,\n            max_position=max_position_embeddings,\n            base=rope_theta,\n        )\n        self.attn = Attention(self.num_heads,\n                              self.head_size,\n                              scaling,\n                              cache_config=cache_config)\n\n    def forward(\n        self,\n        position_ids: torch.Tensor,\n        hidden_states: torch.Tensor,\n        kv_cache: torch.Tensor,\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        qkv, _ = self.qkv_proj(hidden_states)\n        q, k, v = qkv.chunk(chunks=3, dim=-1)\n        q, k = self.rotary_emb(position_ids, q, k)\n        attn_output = self.attn(q, k, v, kv_cache, attn_metadata)\n        output, _ = self.dense(attn_output)\n        return output\n\n\nclass PhiMLP(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n\n        n_inner = getattr(config, \"n_inner\", None)\n        n_inner = n_inner if n_inner is not None else 4 * config.hidden_size\n\n        self.fc1 = ColumnParallelLinear(\n            config.hidden_size,\n            n_inner,\n            quant_config=quant_config,\n        )\n        self.fc2 = RowParallelLinear(\n            n_inner,\n            config.hidden_size,\n            quant_config=quant_config,\n        )\n        self.act = get_act_fn(config.hidden_act, quant_config, n_inner)\n\n    def forward(self, hidden_states):\n        hidden_states, _ = self.fc1(hidden_states)\n        hidden_states = self.act(hidden_states)\n        hidden_states, _ = self.fc2(hidden_states)\n        return hidden_states\n\n\nclass PhiLayer(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 cache_config: Optional[CacheConfig] = None,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n        self.input_layernorm = nn.LayerNorm(config.hidden_size,\n                                            eps=config.layer_norm_eps)\n        self.self_attn = PhiAttention(config, cache_config, quant_config)\n        self.mlp = PhiMLP(config, quant_config)\n\n    def forward(\n        self,\n        position_ids: torch.Tensor,\n        hidden_states: torch.Tensor,\n        kv_cache: torch.Tensor,\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        residual = hidden_states\n        hidden_states = self.input_layernorm(hidden_states)\n        attn_outputs = self.self_attn(\n            position_ids=position_ids,\n            hidden_states=hidden_states,\n            kv_cache=kv_cache,\n            attn_metadata=attn_metadata,\n        )\n        feed_forward_hidden_states = self.mlp(hidden_states)\n        hidden_states = attn_outputs + feed_forward_hidden_states + residual\n        return hidden_states\n\n\nclass PhiModel(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 cache_config: Optional[CacheConfig] = None,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n        self.config = config\n        self.quant_config = quant_config\n        self.embed_tokens = VocabParallelEmbedding(config.vocab_size,\n                                                   config.hidden_size)\n        self.layers = nn.ModuleList([\n            PhiLayer(config, cache_config, quant_config)\n            for _ in range(config.num_hidden_layers)\n        ])\n        self.final_layernorm = nn.LayerNorm(config.hidden_size,\n                                            eps=config.layer_norm_eps)\n\n    def forward(\n        self,\n        input_ids: torch.Tensor,\n        positions: torch.Tensor,\n        kv_caches: List[torch.Tensor],\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        hidden_states = self.embed_tokens(input_ids)\n        for i in range(self.config.num_hidden_layers):\n            layer = self.layers[i]\n            hidden_states = layer(\n                positions,\n                hidden_states,\n                kv_caches[i],\n                attn_metadata,\n            )\n\n        hidden_states = self.final_layernorm(hidden_states)\n\n        return hidden_states\n\n\nclass PhiForCausalLM(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 cache_config: Optional[CacheConfig] = None,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n        self.config = config\n        self.quant_config = quant_config\n\n        self.model = PhiModel(config, cache_config, quant_config)\n\n        self.lm_head = ParallelLMHead(config.vocab_size,\n                                      config.hidden_size,\n                                      bias=True)\n        self.logits_processor = LogitsProcessor(config.vocab_size)\n        self.sampler = Sampler()\n\n    def forward(\n        self,\n        input_ids: torch.Tensor,\n        positions: torch.Tensor,\n        kv_caches: List[torch.Tensor],\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        hidden_states = self.model(input_ids, positions, kv_caches,\n                                   attn_metadata)\n\n        return hidden_states\n\n    def compute_logits(self, hidden_states: torch.Tensor,\n                       sampling_metadata: SamplingMetadata) -> torch.Tensor:\n        logits = self.logits_processor(self.lm_head.weight, hidden_states,\n                                       sampling_metadata, self.lm_head.bias)\n        return logits\n\n    def sample(\n        self,\n        logits: torch.Tensor,\n        sampling_metadata: SamplingMetadata,\n    ) -> Optional[SamplerOutput]:\n        next_tokens = self.sampler(logits, sampling_metadata)\n        return next_tokens\n\n    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):\n        stacked_params_mapping = [\n            # (param_name, shard_name, shard_id)\n            (\"qkv_proj\", \"q_proj\", \"q\"),\n            (\"qkv_proj\", \"k_proj\", \"k\"),\n            (\"qkv_proj\", \"v_proj\", \"v\")\n        ]\n        params_dict = dict(self.named_parameters())\n\n        for name, loaded_weight in weights:\n            if \"rotary_emb.inv_freq\" in name:\n                continue\n\n            for (param_name, weight_name, shard_id) in stacked_params_mapping:\n                if weight_name not in name:\n                    continue\n                name = name.replace(weight_name, param_name)\n                # Skip loading extra bias for GPTQ models.\n                if name.endswith(\".bias\") and name not in params_dict:\n                    continue\n                param = params_dict[name]\n                weight_loader = param.weight_loader\n                weight_loader(param, loaded_weight, shard_id)\n                break\n            else:\n                # Skip loading extra bias for GPTQ models.\n                if name.endswith(\".bias\") and name not in params_dict:\n                    continue\n                # pylint: disable=E1136\n\n                param = params_dict[name]\n                weight_loader = getattr(param, \"weight_loader\",\n                                        default_weight_loader)\n                weight_loader(param, loaded_weight)\n", "path": "vllm/model_executor/models/phi.py"}], "after_files": [{"content": "# coding=utf-8\n# Adapted from\n# https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py\n# Copyright 2023 The vLLM team.\n# Copyright (c) Microsoft Corporation.\n# Licensed under the MIT license.\n#\n# BSD 3-Clause License\n#\n# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.\n# All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are met:\n#\n# * Redistributions of source code must retain the above copyright notice, this\n#   list of conditions and the following disclaimer.\n#\n# * Redistributions in binary form must reproduce the above copyright notice,\n#   this list of conditions and the following disclaimer in the documentation\n#   and/or other materials provided with the distribution.\n#\n# * Neither the name of the copyright holder nor the names of its\n#   contributors may be used to endorse or promote products derived from\n#   this software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\n# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\n# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\"\"\"Inference-only Phi-1.5 model compatible with HuggingFace weights.\"\"\"\nfrom typing import Iterable, List, Optional, Tuple\n\nimport torch\nfrom torch import nn\nfrom transformers import PretrainedConfig\n\nfrom vllm.attention import Attention, AttentionMetadata\nfrom vllm.config import CacheConfig, LoRAConfig\nfrom vllm.distributed import get_tensor_model_parallel_world_size\nfrom vllm.model_executor.layers.activation import get_act_fn\nfrom vllm.model_executor.layers.linear import (ColumnParallelLinear,\n                                               QKVParallelLinear,\n                                               RowParallelLinear)\nfrom vllm.model_executor.layers.logits_processor import LogitsProcessor\nfrom vllm.model_executor.layers.quantization.base_config import (\n    QuantizationConfig)\nfrom vllm.model_executor.layers.rotary_embedding import get_rope\nfrom vllm.model_executor.layers.sampler import Sampler\nfrom vllm.model_executor.layers.vocab_parallel_embedding import (\n    ParallelLMHead, VocabParallelEmbedding)\nfrom vllm.model_executor.model_loader.weight_utils import default_weight_loader\nfrom vllm.model_executor.sampling_metadata import SamplingMetadata\nfrom vllm.sequence import SamplerOutput\n\n\nclass PhiAttention(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 cache_config: Optional[CacheConfig] = None,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n        self.total_num_heads = config.num_attention_heads\n        self.hidden_size = config.hidden_size\n        self.head_size = self.hidden_size // self.total_num_heads\n\n        tensor_model_parallel_world_size = (\n            get_tensor_model_parallel_world_size())\n        assert self.total_num_heads % tensor_model_parallel_world_size == 0\n        self.num_heads = (self.total_num_heads //\n                          tensor_model_parallel_world_size)\n\n        # pylint: disable=C0103\n        self.qkv_proj = QKVParallelLinear(\n            self.hidden_size,\n            self.head_size,\n            self.total_num_heads,\n            bias=True,\n            quant_config=quant_config,\n        )\n        self.dense = RowParallelLinear(\n            self.hidden_size,\n            self.hidden_size,\n            quant_config=quant_config,\n        )\n\n        scaling = self.head_size**-0.5\n        rotary_dim = int(config.partial_rotary_factor *\n                         (config.hidden_size // config.num_attention_heads))\n        assert rotary_dim % 2 == 0\n\n        # pylint: disable=C0301\n        # Refer to:\n        # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518\n        rope_theta = 10000\n        max_position_embeddings = getattr(config, \"n_positions\", 2048)\n        self.rotary_emb = get_rope(\n            self.head_size,\n            rotary_dim=rotary_dim,\n            max_position=max_position_embeddings,\n            base=rope_theta,\n        )\n        self.attn = Attention(self.num_heads,\n                              self.head_size,\n                              scaling,\n                              cache_config=cache_config)\n\n    def forward(\n        self,\n        position_ids: torch.Tensor,\n        hidden_states: torch.Tensor,\n        kv_cache: torch.Tensor,\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        qkv, _ = self.qkv_proj(hidden_states)\n        q, k, v = qkv.chunk(chunks=3, dim=-1)\n        q, k = self.rotary_emb(position_ids, q, k)\n        attn_output = self.attn(q, k, v, kv_cache, attn_metadata)\n        output, _ = self.dense(attn_output)\n        return output\n\n\nclass PhiMLP(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n\n        n_inner = getattr(config, \"n_inner\", None)\n        n_inner = n_inner if n_inner is not None else 4 * config.hidden_size\n\n        self.fc1 = ColumnParallelLinear(\n            config.hidden_size,\n            n_inner,\n            quant_config=quant_config,\n        )\n        self.fc2 = RowParallelLinear(\n            n_inner,\n            config.hidden_size,\n            quant_config=quant_config,\n        )\n        self.act = get_act_fn(config.hidden_act, quant_config, n_inner)\n\n    def forward(self, hidden_states):\n        hidden_states, _ = self.fc1(hidden_states)\n        hidden_states = self.act(hidden_states)\n        hidden_states, _ = self.fc2(hidden_states)\n        return hidden_states\n\n\nclass PhiLayer(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 cache_config: Optional[CacheConfig] = None,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n        self.input_layernorm = nn.LayerNorm(config.hidden_size,\n                                            eps=config.layer_norm_eps)\n        self.self_attn = PhiAttention(config, cache_config, quant_config)\n        self.mlp = PhiMLP(config, quant_config)\n\n    def forward(\n        self,\n        position_ids: torch.Tensor,\n        hidden_states: torch.Tensor,\n        kv_cache: torch.Tensor,\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        residual = hidden_states\n        hidden_states = self.input_layernorm(hidden_states)\n        attn_outputs = self.self_attn(\n            position_ids=position_ids,\n            hidden_states=hidden_states,\n            kv_cache=kv_cache,\n            attn_metadata=attn_metadata,\n        )\n        feed_forward_hidden_states = self.mlp(hidden_states)\n        hidden_states = attn_outputs + feed_forward_hidden_states + residual\n        return hidden_states\n\n\nclass PhiModel(nn.Module):\n\n    def __init__(self,\n                 config: PretrainedConfig,\n                 cache_config: Optional[CacheConfig] = None,\n                 quant_config: Optional[QuantizationConfig] = None):\n        super().__init__()\n        self.config = config\n        self.quant_config = quant_config\n        self.embed_tokens = VocabParallelEmbedding(config.vocab_size,\n                                                   config.hidden_size)\n        self.layers = nn.ModuleList([\n            PhiLayer(config, cache_config, quant_config)\n            for _ in range(config.num_hidden_layers)\n        ])\n        self.final_layernorm = nn.LayerNorm(config.hidden_size,\n                                            eps=config.layer_norm_eps)\n\n    def forward(\n        self,\n        input_ids: torch.Tensor,\n        positions: torch.Tensor,\n        kv_caches: List[torch.Tensor],\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        hidden_states = self.embed_tokens(input_ids)\n        for i in range(self.config.num_hidden_layers):\n            layer = self.layers[i]\n            hidden_states = layer(\n                positions,\n                hidden_states,\n                kv_caches[i],\n                attn_metadata,\n            )\n\n        hidden_states = self.final_layernorm(hidden_states)\n\n        return hidden_states\n\n\nclass PhiForCausalLM(nn.Module):\n    packed_modules_mapping = {\n        \"qkv_proj\": [\n            \"q_proj\",\n            \"k_proj\",\n            \"v_proj\",\n        ]\n    }\n\n    # LoRA specific attributes\n    supported_lora_modules = [\n        \"qkv_proj\",\n        \"dense\",\n        \"fc1\",\n        \"fc2\",\n    ]\n    embedding_modules = {}\n    embedding_padding_modules = []\n\n    def __init__(\n        self,\n        config: PretrainedConfig,\n        cache_config: Optional[CacheConfig] = None,\n        quant_config: Optional[QuantizationConfig] = None,\n        lora_config: Optional[LoRAConfig] = None,\n    ):\n        del lora_config  # Unused.\n        super().__init__()\n        self.config = config\n        self.quant_config = quant_config\n\n        self.model = PhiModel(config, cache_config, quant_config)\n\n        self.lm_head = ParallelLMHead(config.vocab_size,\n                                      config.hidden_size,\n                                      bias=True)\n        self.logits_processor = LogitsProcessor(config.vocab_size)\n        self.sampler = Sampler()\n\n    def forward(\n        self,\n        input_ids: torch.Tensor,\n        positions: torch.Tensor,\n        kv_caches: List[torch.Tensor],\n        attn_metadata: AttentionMetadata,\n    ) -> torch.Tensor:\n        hidden_states = self.model(input_ids, positions, kv_caches,\n                                   attn_metadata)\n\n        return hidden_states\n\n    def compute_logits(self, hidden_states: torch.Tensor,\n                       sampling_metadata: SamplingMetadata) -> torch.Tensor:\n        logits = self.logits_processor(self.lm_head.weight, hidden_states,\n                                       sampling_metadata, self.lm_head.bias)\n        return logits\n\n    def sample(\n        self,\n        logits: torch.Tensor,\n        sampling_metadata: SamplingMetadata,\n    ) -> Optional[SamplerOutput]:\n        next_tokens = self.sampler(logits, sampling_metadata)\n        return next_tokens\n\n    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):\n        stacked_params_mapping = [\n            # (param_name, shard_name, shard_id)\n            (\"qkv_proj\", \"q_proj\", \"q\"),\n            (\"qkv_proj\", \"k_proj\", \"k\"),\n            (\"qkv_proj\", \"v_proj\", \"v\")\n        ]\n        params_dict = dict(self.named_parameters())\n\n        for name, loaded_weight in weights:\n            if \"rotary_emb.inv_freq\" in name:\n                continue\n\n            for (param_name, weight_name, shard_id) in stacked_params_mapping:\n                if weight_name not in name:\n                    continue\n                name = name.replace(weight_name, param_name)\n                # Skip loading extra bias for GPTQ models.\n                if name.endswith(\".bias\") and name not in params_dict:\n                    continue\n                param = params_dict[name]\n                weight_loader = param.weight_loader\n                weight_loader(param, loaded_weight, shard_id)\n                break\n            else:\n                # Skip loading extra bias for GPTQ models.\n                if name.endswith(\".bias\") and name not in params_dict:\n                    continue\n                # pylint: disable=E1136\n\n                param = params_dict[name]\n                weight_loader = getattr(param, \"weight_loader\",\n                                        default_weight_loader)\n                weight_loader(param, loaded_weight)\n", "path": "vllm/model_executor/models/phi.py"}]}