We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
AWQ does not support Turing GPUs
              @WoosukKwon Doesn't it support Turing arch? my GPU's compute capabitlity is 7.5. CUDA-12.1.

build Error message:

  `ptxas /tmp/tmpxft_0006e7c4_00000000-6_gemm_kernels.ptx, line 928; error   : Feature '.m16n8k16' requires .target sm_80 or higher`

If not, hope can add backward compatibility for kernel build.

_Originally posted by @esmeetu in https://github.com/vllm-project/vllm/issues/1032#issuecomment-1722179620_
            

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/model_executor/quantization_utils/awq.py`
Content:
```
1 from typing import Any, Dict, List
2 
3 import torch
4 
5 from vllm.model_executor.quantization_utils.base import QuantizationConfig
6 
7 
8 class AWQConfig(QuantizationConfig):
9     """Config class for AWQ.
10 
11     Reference: https://arxiv.org/abs/2306.00978
12     """
13 
14     def __init__(
15         self,
16         weight_bits: int,
17         group_size: int,
18         zero_point: bool,
19     ) -> None:
20         self.weight_bits = weight_bits
21         self.group_size = group_size
22         self.zero_point = zero_point
23 
24         if self.weight_bits != 4:
25             raise ValueError(
26                 "Currently, only 4-bit weight quantization is supported for "
27                 f"AWQ, but got {self.weight_bits} bits.")
28         self.pack_factor = 32 // self.weight_bits
29 
30     def __repr__(self) -> str:
31         return (f"AWQConfig(weight_bits={self.weight_bits}, "
32                 f"group_size={self.group_size}, "
33                 f"zero_point={self.zero_point})")
34 
35     @classmethod
36     def get_name(cls) -> str:
37         return "awq"
38 
39     @classmethod
40     def get_supported_act_dtypes(cls) -> List[torch.dtype]:
41         return [torch.half]
42 
43     @classmethod
44     def get_config_filenames(cls) -> List[str]:
45         return [
46             "quant_config.json",  # E.g., casperhansen/vicuna-7b-v1.5-awq
47             "quantize_config.json",  # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq  # pylint: disable=line-too-long
48         ]
49 
50     @classmethod
51     def from_config(cls, config: Dict[str, Any]) -> "AWQConfig":
52         weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
53         group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
54         zero_point = cls.get_from_keys(config, ["zero_point"])
55         return cls(weight_bits, group_size, zero_point)
56 
57     @classmethod
58     def get_packed_tensor_names(cls) -> List[str]:
59         return ["qweight", "qzeros"]
60 
61     @classmethod
62     def get_transposed_tensor_names(cls) -> List[str]:
63         return ["qweight", "qzeros", "scales"]
64 
65     @classmethod
66     def get_tp_tensor_names(cls) -> List[str]:
67         return ["qweight", "qzeros", "scales"]
68
```

Path: `vllm/model_executor/quantization_utils/base.py`
Content:
```
1 from typing import Any, Dict, List
2 
3 import torch
4 
5 
6 class QuantizationConfig:
7 
8     @classmethod
9     def get_name(cls) -> str:
10         """Name of the quantization method."""
11         raise NotImplementedError
12 
13     @classmethod
14     def get_supported_act_dtypes(cls) -> List[torch.dtype]:
15         """List of supported activation dtypes."""
16         raise NotImplementedError
17 
18     @classmethod
19     def get_config_filenames(cls) -> List[str]:
20         """List of filenames to search for in the model directory."""
21         raise NotImplementedError
22 
23     @classmethod
24     def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig":
25         """Create a config class from the model's quantization config."""
26         raise NotImplementedError
27 
28     @staticmethod
29     def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
30         """Get a value from the model's quantization config."""
31         for key in keys:
32             if key in config:
33                 return config[key]
34         raise ValueError(f"Cannot find any of {keys} in the model's "
35                          "quantization config.")
36 
37     @classmethod
38     def get_packed_tensor_names(cls) -> List[str]:
39         raise NotImplementedError
40 
41     @classmethod
42     def is_packed(cls, tensor_name: str) -> bool:
43         """Returns True if a tensor is packed.
44 
45         A tensor is considered packed if each element in the tensor is a
46         packed representation of multiple elements in the original tensor.
47         For example, an INT32 element in the tensor may represent 8 INT4
48         elements in the original tensor.
49         """
50         return any(tag in tensor_name for tag in cls.get_packed_tensor_names())
51 
52     @classmethod
53     def get_transposed_tensor_names(cls) -> List[str]:
54         raise NotImplementedError
55 
56     @classmethod
57     def is_transposed(cls, tensor_name: str) -> bool:
58         """Returns True if a tensor is transposed relative to nn.Linear.weight.
59         """
60         return any(tag in tensor_name
61                    for tag in cls.get_transposed_tensor_names())
62 
63     @classmethod
64     def get_tp_tensor_names(cls) -> List[str]:
65         raise NotImplementedError
66
```

Path: `vllm/model_executor/model_loader.py`
Content:
```
1 """Utilities for selecting and loading models."""
2 import contextlib
3 from typing import Type
4 
5 import torch
6 import torch.nn as nn
7 from transformers import PretrainedConfig
8 
9 from vllm.config import ModelConfig
10 from vllm.model_executor.models import *  # pylint: disable=wildcard-import
11 from vllm.model_executor.weight_utils import (get_quant_config,
12                                               initialize_dummy_weights)
13 
14 # TODO(woosuk): Lazy-load the model classes.
15 _MODEL_REGISTRY = {
16     "AquilaModel": AquilaForCausalLM,
17     "BaiChuanForCausalLM": BaiChuanForCausalLM,  # baichuan-7b
18     "BaichuanForCausalLM": BaichuanForCausalLM,  # baichuan-13b
19     "BloomForCausalLM": BloomForCausalLM,
20     "FalconForCausalLM": FalconForCausalLM,
21     "GPT2LMHeadModel": GPT2LMHeadModel,
22     "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
23     "GPTJForCausalLM": GPTJForCausalLM,
24     "GPTNeoXForCausalLM": GPTNeoXForCausalLM,
25     "InternLMForCausalLM": InternLMForCausalLM,
26     "LlamaForCausalLM": LlamaForCausalLM,
27     "LLaMAForCausalLM": LlamaForCausalLM,  # For decapoda-research/llama-*
28     "MPTForCausalLM": MPTForCausalLM,
29     "OPTForCausalLM": OPTForCausalLM,
30     "QWenLMHeadModel": QWenLMHeadModel,
31     "RWForCausalLM": FalconForCausalLM,
32 }
33 
34 # FIXME(woosuk): Remove this once all models support quantization.
35 _MODEL_CLASSES_SUPPORT_QUANTIZATION = [
36     LlamaForCausalLM,
37 ]
38 
39 
40 @contextlib.contextmanager
41 def _set_default_torch_dtype(dtype: torch.dtype):
42     """Sets the default torch dtype to the given dtype."""
43     old_dtype = torch.get_default_dtype()
44     torch.set_default_dtype(dtype)
45     yield
46     torch.set_default_dtype(old_dtype)
47 
48 
49 def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
50     architectures = getattr(config, "architectures", [])
51     for arch in architectures:
52         if arch in _MODEL_REGISTRY:
53             return _MODEL_REGISTRY[arch]
54     raise ValueError(
55         f"Model architectures {architectures} are not supported for now. "
56         f"Supported architectures: {list(_MODEL_REGISTRY.keys())}")
57 
58 
59 def get_model(model_config: ModelConfig) -> nn.Module:
60     model_class = _get_model_architecture(model_config.hf_config)
61 
62     # Get the quantization config.
63     quant_config = None
64     if model_config.quantization is not None:
65         if model_class not in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
66             raise ValueError(
67                 f"Quantization is not supported for {model_class}.")
68         quant_config = get_quant_config(model_config.quantization,
69                                         model_config.model,
70                                         model_config.download_dir)
71         supported_dtypes = quant_config.get_supported_act_dtypes()
72         if model_config.dtype not in supported_dtypes:
73             raise ValueError(
74                 f"{model_config.dtype} is not supported for quantization "
75                 f"method {model_config.quantization}. Supported dtypes: "
76                 f"{supported_dtypes}")
77 
78     with _set_default_torch_dtype(model_config.dtype):
79         # Create a model instance.
80         # The weights will be initialized as empty tensors.
81         if model_class in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
82             model = model_class(model_config.hf_config, quant_config)
83         else:
84             model = model_class(model_config.hf_config)
85         if model_config.load_format == "dummy":
86             model = model.cuda()
87             # NOTE(woosuk): For accurate performance evaluation, we assign
88             # random values to the weights.
89             initialize_dummy_weights(model)
90         else:
91             # Load the weights from the cached or downloaded files.
92             model.load_weights(model_config.model, model_config.download_dir,
93                                model_config.load_format, model_config.revision)
94             model = model.cuda()
95     return model.eval()
96
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/model_executor/model_loader.py b/vllm/model_executor/model_loader.py
--- a/vllm/model_executor/model_loader.py
+++ b/vllm/model_executor/model_loader.py
@@ -68,6 +68,14 @@
         quant_config = get_quant_config(model_config.quantization,
                                         model_config.model,
                                         model_config.download_dir)
+        capability = torch.cuda.get_device_capability()
+        capability = capability[0] * 10 + capability[1]
+        if capability < quant_config.get_min_capability():
+            raise ValueError(
+                f"The quantization method {model_config.quantization} is not "
+                "supported for the current GPU. "
+                f"Minimum capability: {quant_config.get_min_capability()}. "
+                f"Current capability: {capability}.")
         supported_dtypes = quant_config.get_supported_act_dtypes()
         if model_config.dtype not in supported_dtypes:
             raise ValueError(
diff --git a/vllm/model_executor/quantization_utils/awq.py b/vllm/model_executor/quantization_utils/awq.py
--- a/vllm/model_executor/quantization_utils/awq.py
+++ b/vllm/model_executor/quantization_utils/awq.py
@@ -40,6 +40,11 @@
     def get_supported_act_dtypes(cls) -> List[torch.dtype]:
         return [torch.half]
 
+    @classmethod
+    def get_min_capability(cls) -> int:
+        # The AWQ kernel only supports Ampere or newer GPUs.
+        return 80
+
     @classmethod
     def get_config_filenames(cls) -> List[str]:
         return [
diff --git a/vllm/model_executor/quantization_utils/base.py b/vllm/model_executor/quantization_utils/base.py
--- a/vllm/model_executor/quantization_utils/base.py
+++ b/vllm/model_executor/quantization_utils/base.py
@@ -15,6 +15,16 @@
         """List of supported activation dtypes."""
         raise NotImplementedError
 
+    @classmethod
+    def get_min_capability(cls) -> int:
+        """Minimum GPU capability to support the quantization method.
+
+        E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
+        This requirement is due to the custom CUDA kernels used by the
+        quantization method.
+        """
+        raise NotImplementedError
+
     @classmethod
     def get_config_filenames(cls) -> List[str]:
         """List of filenames to search for in the model directory."""

{"golden_diff": "diff --git a/vllm/model_executor/model_loader.py b/vllm/model_executor/model_loader.py\n--- a/vllm/model_executor/model_loader.py\n+++ b/vllm/model_executor/model_loader.py\n@@ -68,6 +68,14 @@\n         quant_config = get_quant_config(model_config.quantization,\n                                         model_config.model,\n                                         model_config.download_dir)\n+        capability = torch.cuda.get_device_capability()\n+        capability = capability[0] * 10 + capability[1]\n+        if capability < quant_config.get_min_capability():\n+            raise ValueError(\n+                f\"The quantization method {model_config.quantization} is not \"\n+                \"supported for the current GPU. \"\n+                f\"Minimum capability: {quant_config.get_min_capability()}. \"\n+                f\"Current capability: {capability}.\")\n         supported_dtypes = quant_config.get_supported_act_dtypes()\n         if model_config.dtype not in supported_dtypes:\n             raise ValueError(\ndiff --git a/vllm/model_executor/quantization_utils/awq.py b/vllm/model_executor/quantization_utils/awq.py\n--- a/vllm/model_executor/quantization_utils/awq.py\n+++ b/vllm/model_executor/quantization_utils/awq.py\n@@ -40,6 +40,11 @@\n     def get_supported_act_dtypes(cls) -> List[torch.dtype]:\n         return [torch.half]\n \n+    @classmethod\n+    def get_min_capability(cls) -> int:\n+        # The AWQ kernel only supports Ampere or newer GPUs.\n+        return 80\n+\n     @classmethod\n     def get_config_filenames(cls) -> List[str]:\n         return [\ndiff --git a/vllm/model_executor/quantization_utils/base.py b/vllm/model_executor/quantization_utils/base.py\n--- a/vllm/model_executor/quantization_utils/base.py\n+++ b/vllm/model_executor/quantization_utils/base.py\n@@ -15,6 +15,16 @@\n         \"\"\"List of supported activation dtypes.\"\"\"\n         raise NotImplementedError\n \n+    @classmethod\n+    def get_min_capability(cls) -> int:\n+        \"\"\"Minimum GPU capability to support the quantization method.\n+\n+        E.g., 70 for Volta, 75 for Turing, 80 for Ampere.\n+        This requirement is due to the custom CUDA kernels used by the\n+        quantization method.\n+        \"\"\"\n+        raise NotImplementedError\n+\n     @classmethod\n     def get_config_filenames(cls) -> List[str]:\n         \"\"\"List of filenames to search for in the model directory.\"\"\"\n", "issue": "AWQ does not support Turing GPUs\n              @WoosukKwon Doesn't it support Turing arch? my GPU's compute capabitlity is 7.5. CUDA-12.1.\r\n\r\nbuild Error message:\r\n\r\n  `ptxas /tmp/tmpxft_0006e7c4_00000000-6_gemm_kernels.ptx, line 928; error   : Feature '.m16n8k16' requires .target sm_80 or higher`\r\n\r\nIf not, hope can add backward compatibility for kernel build.\r\n\r\n_Originally posted by @esmeetu in https://github.com/vllm-project/vllm/issues/1032#issuecomment-1722179620_\r\n            \n", "before_files": [{"content": "from typing import Any, Dict, List\n\nimport torch\n\nfrom vllm.model_executor.quantization_utils.base import QuantizationConfig\n\n\nclass AWQConfig(QuantizationConfig):\n    \"\"\"Config class for AWQ.\n\n    Reference: https://arxiv.org/abs/2306.00978\n    \"\"\"\n\n    def __init__(\n        self,\n        weight_bits: int,\n        group_size: int,\n        zero_point: bool,\n    ) -> None:\n        self.weight_bits = weight_bits\n        self.group_size = group_size\n        self.zero_point = zero_point\n\n        if self.weight_bits != 4:\n            raise ValueError(\n                \"Currently, only 4-bit weight quantization is supported for \"\n                f\"AWQ, but got {self.weight_bits} bits.\")\n        self.pack_factor = 32 // self.weight_bits\n\n    def __repr__(self) -> str:\n        return (f\"AWQConfig(weight_bits={self.weight_bits}, \"\n                f\"group_size={self.group_size}, \"\n                f\"zero_point={self.zero_point})\")\n\n    @classmethod\n    def get_name(cls) -> str:\n        return \"awq\"\n\n    @classmethod\n    def get_supported_act_dtypes(cls) -> List[torch.dtype]:\n        return [torch.half]\n\n    @classmethod\n    def get_config_filenames(cls) -> List[str]:\n        return [\n            \"quant_config.json\",  # E.g., casperhansen/vicuna-7b-v1.5-awq\n            \"quantize_config.json\",  # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq  # pylint: disable=line-too-long\n        ]\n\n    @classmethod\n    def from_config(cls, config: Dict[str, Any]) -> \"AWQConfig\":\n        weight_bits = cls.get_from_keys(config, [\"w_bit\", \"bits\"])\n        group_size = cls.get_from_keys(config, [\"q_group_size\", \"group_size\"])\n        zero_point = cls.get_from_keys(config, [\"zero_point\"])\n        return cls(weight_bits, group_size, zero_point)\n\n    @classmethod\n    def get_packed_tensor_names(cls) -> List[str]:\n        return [\"qweight\", \"qzeros\"]\n\n    @classmethod\n    def get_transposed_tensor_names(cls) -> List[str]:\n        return [\"qweight\", \"qzeros\", \"scales\"]\n\n    @classmethod\n    def get_tp_tensor_names(cls) -> List[str]:\n        return [\"qweight\", \"qzeros\", \"scales\"]\n", "path": "vllm/model_executor/quantization_utils/awq.py"}, {"content": "from typing import Any, Dict, List\n\nimport torch\n\n\nclass QuantizationConfig:\n\n    @classmethod\n    def get_name(cls) -> str:\n        \"\"\"Name of the quantization method.\"\"\"\n        raise NotImplementedError\n\n    @classmethod\n    def get_supported_act_dtypes(cls) -> List[torch.dtype]:\n        \"\"\"List of supported activation dtypes.\"\"\"\n        raise NotImplementedError\n\n    @classmethod\n    def get_config_filenames(cls) -> List[str]:\n        \"\"\"List of filenames to search for in the model directory.\"\"\"\n        raise NotImplementedError\n\n    @classmethod\n    def from_config(cls, config: Dict[str, Any]) -> \"QuantizationConfig\":\n        \"\"\"Create a config class from the model's quantization config.\"\"\"\n        raise NotImplementedError\n\n    @staticmethod\n    def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:\n        \"\"\"Get a value from the model's quantization config.\"\"\"\n        for key in keys:\n            if key in config:\n                return config[key]\n        raise ValueError(f\"Cannot find any of {keys} in the model's \"\n                         \"quantization config.\")\n\n    @classmethod\n    def get_packed_tensor_names(cls) -> List[str]:\n        raise NotImplementedError\n\n    @classmethod\n    def is_packed(cls, tensor_name: str) -> bool:\n        \"\"\"Returns True if a tensor is packed.\n\n        A tensor is considered packed if each element in the tensor is a\n        packed representation of multiple elements in the original tensor.\n        For example, an INT32 element in the tensor may represent 8 INT4\n        elements in the original tensor.\n        \"\"\"\n        return any(tag in tensor_name for tag in cls.get_packed_tensor_names())\n\n    @classmethod\n    def get_transposed_tensor_names(cls) -> List[str]:\n        raise NotImplementedError\n\n    @classmethod\n    def is_transposed(cls, tensor_name: str) -> bool:\n        \"\"\"Returns True if a tensor is transposed relative to nn.Linear.weight.\n        \"\"\"\n        return any(tag in tensor_name\n                   for tag in cls.get_transposed_tensor_names())\n\n    @classmethod\n    def get_tp_tensor_names(cls) -> List[str]:\n        raise NotImplementedError\n", "path": "vllm/model_executor/quantization_utils/base.py"}, {"content": "\"\"\"Utilities for selecting and loading models.\"\"\"\nimport contextlib\nfrom typing import Type\n\nimport torch\nimport torch.nn as nn\nfrom transformers import PretrainedConfig\n\nfrom vllm.config import ModelConfig\nfrom vllm.model_executor.models import *  # pylint: disable=wildcard-import\nfrom vllm.model_executor.weight_utils import (get_quant_config,\n                                              initialize_dummy_weights)\n\n# TODO(woosuk): Lazy-load the model classes.\n_MODEL_REGISTRY = {\n    \"AquilaModel\": AquilaForCausalLM,\n    \"BaiChuanForCausalLM\": BaiChuanForCausalLM,  # baichuan-7b\n    \"BaichuanForCausalLM\": BaichuanForCausalLM,  # baichuan-13b\n    \"BloomForCausalLM\": BloomForCausalLM,\n    \"FalconForCausalLM\": FalconForCausalLM,\n    \"GPT2LMHeadModel\": GPT2LMHeadModel,\n    \"GPTBigCodeForCausalLM\": GPTBigCodeForCausalLM,\n    \"GPTJForCausalLM\": GPTJForCausalLM,\n    \"GPTNeoXForCausalLM\": GPTNeoXForCausalLM,\n    \"InternLMForCausalLM\": InternLMForCausalLM,\n    \"LlamaForCausalLM\": LlamaForCausalLM,\n    \"LLaMAForCausalLM\": LlamaForCausalLM,  # For decapoda-research/llama-*\n    \"MPTForCausalLM\": MPTForCausalLM,\n    \"OPTForCausalLM\": OPTForCausalLM,\n    \"QWenLMHeadModel\": QWenLMHeadModel,\n    \"RWForCausalLM\": FalconForCausalLM,\n}\n\n# FIXME(woosuk): Remove this once all models support quantization.\n_MODEL_CLASSES_SUPPORT_QUANTIZATION = [\n    LlamaForCausalLM,\n]\n\n\n@contextlib.contextmanager\ndef _set_default_torch_dtype(dtype: torch.dtype):\n    \"\"\"Sets the default torch dtype to the given dtype.\"\"\"\n    old_dtype = torch.get_default_dtype()\n    torch.set_default_dtype(dtype)\n    yield\n    torch.set_default_dtype(old_dtype)\n\n\ndef _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:\n    architectures = getattr(config, \"architectures\", [])\n    for arch in architectures:\n        if arch in _MODEL_REGISTRY:\n            return _MODEL_REGISTRY[arch]\n    raise ValueError(\n        f\"Model architectures {architectures} are not supported for now. \"\n        f\"Supported architectures: {list(_MODEL_REGISTRY.keys())}\")\n\n\ndef get_model(model_config: ModelConfig) -> nn.Module:\n    model_class = _get_model_architecture(model_config.hf_config)\n\n    # Get the quantization config.\n    quant_config = None\n    if model_config.quantization is not None:\n        if model_class not in _MODEL_CLASSES_SUPPORT_QUANTIZATION:\n            raise ValueError(\n                f\"Quantization is not supported for {model_class}.\")\n        quant_config = get_quant_config(model_config.quantization,\n                                        model_config.model,\n                                        model_config.download_dir)\n        supported_dtypes = quant_config.get_supported_act_dtypes()\n        if model_config.dtype not in supported_dtypes:\n            raise ValueError(\n                f\"{model_config.dtype} is not supported for quantization \"\n                f\"method {model_config.quantization}. Supported dtypes: \"\n                f\"{supported_dtypes}\")\n\n    with _set_default_torch_dtype(model_config.dtype):\n        # Create a model instance.\n        # The weights will be initialized as empty tensors.\n        if model_class in _MODEL_CLASSES_SUPPORT_QUANTIZATION:\n            model = model_class(model_config.hf_config, quant_config)\n        else:\n            model = model_class(model_config.hf_config)\n        if model_config.load_format == \"dummy\":\n            model = model.cuda()\n            # NOTE(woosuk): For accurate performance evaluation, we assign\n            # random values to the weights.\n            initialize_dummy_weights(model)\n        else:\n            # Load the weights from the cached or downloaded files.\n            model.load_weights(model_config.model, model_config.download_dir,\n                               model_config.load_format, model_config.revision)\n            model = model.cuda()\n    return model.eval()\n", "path": "vllm/model_executor/model_loader.py"}], "after_files": [{"content": "from typing import Any, Dict, List\n\nimport torch\n\nfrom vllm.model_executor.quantization_utils.base import QuantizationConfig\n\n\nclass AWQConfig(QuantizationConfig):\n    \"\"\"Config class for AWQ.\n\n    Reference: https://arxiv.org/abs/2306.00978\n    \"\"\"\n\n    def __init__(\n        self,\n        weight_bits: int,\n        group_size: int,\n        zero_point: bool,\n    ) -> None:\n        self.weight_bits = weight_bits\n        self.group_size = group_size\n        self.zero_point = zero_point\n\n        if self.weight_bits != 4:\n            raise ValueError(\n                \"Currently, only 4-bit weight quantization is supported for \"\n                f\"AWQ, but got {self.weight_bits} bits.\")\n        self.pack_factor = 32 // self.weight_bits\n\n    def __repr__(self) -> str:\n        return (f\"AWQConfig(weight_bits={self.weight_bits}, \"\n                f\"group_size={self.group_size}, \"\n                f\"zero_point={self.zero_point})\")\n\n    @classmethod\n    def get_name(cls) -> str:\n        return \"awq\"\n\n    @classmethod\n    def get_supported_act_dtypes(cls) -> List[torch.dtype]:\n        return [torch.half]\n\n    @classmethod\n    def get_min_capability(cls) -> int:\n        # The AWQ kernel only supports Ampere or newer GPUs.\n        return 80\n\n    @classmethod\n    def get_config_filenames(cls) -> List[str]:\n        return [\n            \"quant_config.json\",  # E.g., casperhansen/vicuna-7b-v1.5-awq\n            \"quantize_config.json\",  # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq  # pylint: disable=line-too-long\n        ]\n\n    @classmethod\n    def from_config(cls, config: Dict[str, Any]) -> \"AWQConfig\":\n        weight_bits = cls.get_from_keys(config, [\"w_bit\", \"bits\"])\n        group_size = cls.get_from_keys(config, [\"q_group_size\", \"group_size\"])\n        zero_point = cls.get_from_keys(config, [\"zero_point\"])\n        return cls(weight_bits, group_size, zero_point)\n\n    @classmethod\n    def get_packed_tensor_names(cls) -> List[str]:\n        return [\"qweight\", \"qzeros\"]\n\n    @classmethod\n    def get_transposed_tensor_names(cls) -> List[str]:\n        return [\"qweight\", \"qzeros\", \"scales\"]\n\n    @classmethod\n    def get_tp_tensor_names(cls) -> List[str]:\n        return [\"qweight\", \"qzeros\", \"scales\"]\n", "path": "vllm/model_executor/quantization_utils/awq.py"}, {"content": "from typing import Any, Dict, List\n\nimport torch\n\n\nclass QuantizationConfig:\n\n    @classmethod\n    def get_name(cls) -> str:\n        \"\"\"Name of the quantization method.\"\"\"\n        raise NotImplementedError\n\n    @classmethod\n    def get_supported_act_dtypes(cls) -> List[torch.dtype]:\n        \"\"\"List of supported activation dtypes.\"\"\"\n        raise NotImplementedError\n\n    @classmethod\n    def get_min_capability(cls) -> int:\n        \"\"\"Minimum GPU capability to support the quantization method.\n\n        E.g., 70 for Volta, 75 for Turing, 80 for Ampere.\n        This requirement is due to the custom CUDA kernels used by the\n        quantization method.\n        \"\"\"\n        raise NotImplementedError\n\n    @classmethod\n    def get_config_filenames(cls) -> List[str]:\n        \"\"\"List of filenames to search for in the model directory.\"\"\"\n        raise NotImplementedError\n\n    @classmethod\n    def from_config(cls, config: Dict[str, Any]) -> \"QuantizationConfig\":\n        \"\"\"Create a config class from the model's quantization config.\"\"\"\n        raise NotImplementedError\n\n    @staticmethod\n    def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:\n        \"\"\"Get a value from the model's quantization config.\"\"\"\n        for key in keys:\n            if key in config:\n                return config[key]\n        raise ValueError(f\"Cannot find any of {keys} in the model's \"\n                         \"quantization config.\")\n\n    @classmethod\n    def get_packed_tensor_names(cls) -> List[str]:\n        raise NotImplementedError\n\n    @classmethod\n    def is_packed(cls, tensor_name: str) -> bool:\n        \"\"\"Returns True if a tensor is packed.\n\n        A tensor is considered packed if each element in the tensor is a\n        packed representation of multiple elements in the original tensor.\n        For example, an INT32 element in the tensor may represent 8 INT4\n        elements in the original tensor.\n        \"\"\"\n        return any(tag in tensor_name for tag in cls.get_packed_tensor_names())\n\n    @classmethod\n    def get_transposed_tensor_names(cls) -> List[str]:\n        raise NotImplementedError\n\n    @classmethod\n    def is_transposed(cls, tensor_name: str) -> bool:\n        \"\"\"Returns True if a tensor is transposed relative to nn.Linear.weight.\n        \"\"\"\n        return any(tag in tensor_name\n                   for tag in cls.get_transposed_tensor_names())\n\n    @classmethod\n    def get_tp_tensor_names(cls) -> List[str]:\n        raise NotImplementedError\n", "path": "vllm/model_executor/quantization_utils/base.py"}, {"content": "\"\"\"Utilities for selecting and loading models.\"\"\"\nimport contextlib\nfrom typing import Type\n\nimport torch\nimport torch.nn as nn\nfrom transformers import PretrainedConfig\n\nfrom vllm.config import ModelConfig\nfrom vllm.model_executor.models import *  # pylint: disable=wildcard-import\nfrom vllm.model_executor.weight_utils import (get_quant_config,\n                                              initialize_dummy_weights)\n\n# TODO(woosuk): Lazy-load the model classes.\n_MODEL_REGISTRY = {\n    \"AquilaModel\": AquilaForCausalLM,\n    \"BaiChuanForCausalLM\": BaiChuanForCausalLM,  # baichuan-7b\n    \"BaichuanForCausalLM\": BaichuanForCausalLM,  # baichuan-13b\n    \"BloomForCausalLM\": BloomForCausalLM,\n    \"FalconForCausalLM\": FalconForCausalLM,\n    \"GPT2LMHeadModel\": GPT2LMHeadModel,\n    \"GPTBigCodeForCausalLM\": GPTBigCodeForCausalLM,\n    \"GPTJForCausalLM\": GPTJForCausalLM,\n    \"GPTNeoXForCausalLM\": GPTNeoXForCausalLM,\n    \"InternLMForCausalLM\": InternLMForCausalLM,\n    \"LlamaForCausalLM\": LlamaForCausalLM,\n    \"LLaMAForCausalLM\": LlamaForCausalLM,  # For decapoda-research/llama-*\n    \"MPTForCausalLM\": MPTForCausalLM,\n    \"OPTForCausalLM\": OPTForCausalLM,\n    \"QWenLMHeadModel\": QWenLMHeadModel,\n    \"RWForCausalLM\": FalconForCausalLM,\n}\n\n# FIXME(woosuk): Remove this once all models support quantization.\n_MODEL_CLASSES_SUPPORT_QUANTIZATION = [\n    LlamaForCausalLM,\n]\n\n\n@contextlib.contextmanager\ndef _set_default_torch_dtype(dtype: torch.dtype):\n    \"\"\"Sets the default torch dtype to the given dtype.\"\"\"\n    old_dtype = torch.get_default_dtype()\n    torch.set_default_dtype(dtype)\n    yield\n    torch.set_default_dtype(old_dtype)\n\n\ndef _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:\n    architectures = getattr(config, \"architectures\", [])\n    for arch in architectures:\n        if arch in _MODEL_REGISTRY:\n            return _MODEL_REGISTRY[arch]\n    raise ValueError(\n        f\"Model architectures {architectures} are not supported for now. \"\n        f\"Supported architectures: {list(_MODEL_REGISTRY.keys())}\")\n\n\ndef get_model(model_config: ModelConfig) -> nn.Module:\n    model_class = _get_model_architecture(model_config.hf_config)\n\n    # Get the quantization config.\n    quant_config = None\n    if model_config.quantization is not None:\n        if model_class not in _MODEL_CLASSES_SUPPORT_QUANTIZATION:\n            raise ValueError(\n                f\"Quantization is not supported for {model_class}.\")\n        quant_config = get_quant_config(model_config.quantization,\n                                        model_config.model,\n                                        model_config.download_dir)\n        capability = torch.cuda.get_device_capability()\n        capability = capability[0] * 10 + capability[1]\n        if capability < quant_config.get_min_capability():\n            raise ValueError(\n                f\"The quantization method {model_config.quantization} is not \"\n                \"supported for the current GPU. \"\n                f\"Minimum capability: {quant_config.get_min_capability()}. \"\n                f\"Current capability: {capability}.\")\n        supported_dtypes = quant_config.get_supported_act_dtypes()\n        if model_config.dtype not in supported_dtypes:\n            raise ValueError(\n                f\"{model_config.dtype} is not supported for quantization \"\n                f\"method {model_config.quantization}. Supported dtypes: \"\n                f\"{supported_dtypes}\")\n\n    with _set_default_torch_dtype(model_config.dtype):\n        # Create a model instance.\n        # The weights will be initialized as empty tensors.\n        if model_class in _MODEL_CLASSES_SUPPORT_QUANTIZATION:\n            model = model_class(model_config.hf_config, quant_config)\n        else:\n            model = model_class(model_config.hf_config)\n        if model_config.load_format == \"dummy\":\n            model = model.cuda()\n            # NOTE(woosuk): For accurate performance evaluation, we assign\n            # random values to the weights.\n            initialize_dummy_weights(model)\n        else:\n            # Load the weights from the cached or downloaded files.\n            model.load_weights(model_config.model, model_config.download_dir,\n                               model_config.load_format, model_config.revision)\n            model = model.cuda()\n    return model.eval()\n", "path": "vllm/model_executor/model_loader.py"}]}