We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Installation Error
When I install vLLM, I have pull the docker as the tutorial and install from source. But there are an error encountered, how can I fix this:
`ValueError: Unsupported CUDA arch (5.2). Valid CUDA arch strings are: ['7.0', '7.5', '8.0', '8.6', '8.9', '9.0', '7.0+PTX', '7.5+PTX', '8.0+PTX', '8.6+PTX', '8.9+PTX', '9.0+PTX'].`

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `setup.py`
Content:
```
1 import io
2 import os
3 import re
4 import subprocess
5 from typing import List, Set
6 import warnings
7 
8 from packaging.version import parse, Version
9 import setuptools
10 import torch
11 from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
12 
13 ROOT_DIR = os.path.dirname(__file__)
14 
15 # Supported NVIDIA GPU architectures.
16 SUPPORTED_ARCHS = ["7.0", "7.5", "8.0", "8.6", "8.9", "9.0"]
17 
18 # Compiler flags.
19 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
20 # TODO(woosuk): Should we use -O3?
21 NVCC_FLAGS = ["-O2", "-std=c++17"]
22 
23 ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
24 CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
25 NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
26 
27 if CUDA_HOME is None:
28     raise RuntimeError(
29         "Cannot find CUDA_HOME. CUDA must be available to build the package.")
30 
31 
32 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
33     """Get the CUDA version from nvcc.
34 
35     Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
36     """
37     nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
38                                           universal_newlines=True)
39     output = nvcc_output.split()
40     release_idx = output.index("release") + 1
41     nvcc_cuda_version = parse(output[release_idx].split(",")[0])
42     return nvcc_cuda_version
43 
44 
45 def get_torch_arch_list() -> Set[str]:
46     # TORCH_CUDA_ARCH_LIST can have one or more architectures,
47     # e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
48     # compiler to additionally include PTX code that can be runtime-compiled
49     # and executed on the 8.6 or newer architectures. While the PTX code will
50     # not give the best performance on the newer architectures, it provides
51     # forward compatibility.
52     valid_arch_strs = SUPPORTED_ARCHS + [s + "+PTX" for s in SUPPORTED_ARCHS]
53     arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
54     if arch_list is None:
55         return set()
56 
57     # List are separated by ; or space.
58     arch_list = arch_list.replace(" ", ";").split(";")
59     for arch in arch_list:
60         if arch not in valid_arch_strs:
61             raise ValueError(
62                 f"Unsupported CUDA arch ({arch}). "
63                 f"Valid CUDA arch strings are: {valid_arch_strs}.")
64     return set(arch_list)
65 
66 
67 # First, check the TORCH_CUDA_ARCH_LIST environment variable.
68 compute_capabilities = get_torch_arch_list()
69 if not compute_capabilities:
70     # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
71     # GPUs on the current machine.
72     device_count = torch.cuda.device_count()
73     for i in range(device_count):
74         major, minor = torch.cuda.get_device_capability(i)
75         if major < 7:
76             raise RuntimeError(
77                 "GPUs with compute capability below 7.0 are not supported.")
78         compute_capabilities.add(f"{major}.{minor}")
79 
80 nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
81 if not compute_capabilities:
82     # If no GPU is specified nor available, add all supported architectures
83     # based on the NVCC CUDA version.
84     compute_capabilities = set(SUPPORTED_ARCHS)
85     if nvcc_cuda_version < Version("11.1"):
86         compute_capabilities.remove("8.6")
87     if nvcc_cuda_version < Version("11.8"):
88         compute_capabilities.remove("8.9")
89         compute_capabilities.remove("9.0")
90 
91 # Validate the NVCC CUDA version.
92 if nvcc_cuda_version < Version("11.0"):
93     raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
94 if nvcc_cuda_version < Version("11.1"):
95     if any(cc.startswith("8.6") for cc in compute_capabilities):
96         raise RuntimeError(
97             "CUDA 11.1 or higher is required for compute capability 8.6.")
98 if nvcc_cuda_version < Version("11.8"):
99     if any(cc.startswith("8.9") for cc in compute_capabilities):
100         # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
101         # However, GPUs with compute capability 8.9 can also run the code generated by
102         # the previous versions of CUDA 11 and targeting compute capability 8.0.
103         # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
104         # instead of 8.9.
105         warnings.warn(
106             "CUDA 11.8 or higher is required for compute capability 8.9. "
107             "Targeting compute capability 8.0 instead.")
108         compute_capabilities = set(cc for cc in compute_capabilities
109                                    if not cc.startswith("8.9"))
110         compute_capabilities.add("8.0+PTX")
111     if any(cc.startswith("9.0") for cc in compute_capabilities):
112         raise RuntimeError(
113             "CUDA 11.8 or higher is required for compute capability 9.0.")
114 
115 # Add target compute capabilities to NVCC flags.
116 for capability in compute_capabilities:
117     num = capability[0] + capability[2]
118     NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
119     if capability.endswith("+PTX"):
120         NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=compute_{num}"]
121 
122 # Use NVCC threads to parallelize the build.
123 if nvcc_cuda_version >= Version("11.2"):
124     num_threads = min(os.cpu_count(), 8)
125     NVCC_FLAGS += ["--threads", str(num_threads)]
126 
127 ext_modules = []
128 
129 # Cache operations.
130 cache_extension = CUDAExtension(
131     name="vllm.cache_ops",
132     sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
133     extra_compile_args={
134         "cxx": CXX_FLAGS,
135         "nvcc": NVCC_FLAGS,
136     },
137 )
138 ext_modules.append(cache_extension)
139 
140 # Attention kernels.
141 attention_extension = CUDAExtension(
142     name="vllm.attention_ops",
143     sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
144     extra_compile_args={
145         "cxx": CXX_FLAGS,
146         "nvcc": NVCC_FLAGS,
147     },
148 )
149 ext_modules.append(attention_extension)
150 
151 # Positional encoding kernels.
152 positional_encoding_extension = CUDAExtension(
153     name="vllm.pos_encoding_ops",
154     sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
155     extra_compile_args={
156         "cxx": CXX_FLAGS,
157         "nvcc": NVCC_FLAGS,
158     },
159 )
160 ext_modules.append(positional_encoding_extension)
161 
162 # Layer normalization kernels.
163 layernorm_extension = CUDAExtension(
164     name="vllm.layernorm_ops",
165     sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
166     extra_compile_args={
167         "cxx": CXX_FLAGS,
168         "nvcc": NVCC_FLAGS,
169     },
170 )
171 ext_modules.append(layernorm_extension)
172 
173 # Activation kernels.
174 activation_extension = CUDAExtension(
175     name="vllm.activation_ops",
176     sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
177     extra_compile_args={
178         "cxx": CXX_FLAGS,
179         "nvcc": NVCC_FLAGS,
180     },
181 )
182 ext_modules.append(activation_extension)
183 
184 # Quantization kernels.
185 quantization_extension = CUDAExtension(
186     name="vllm.quantization_ops",
187     sources=[
188         "csrc/quantization.cpp",
189         "csrc/quantization/awq/gemm_kernels.cu",
190     ],
191     extra_compile_args={
192         "cxx": CXX_FLAGS,
193         "nvcc": NVCC_FLAGS,
194     },
195 )
196 ext_modules.append(quantization_extension)
197 
198 # Misc. CUDA utils.
199 cuda_utils_extension = CUDAExtension(
200     name="vllm.cuda_utils",
201     sources=["csrc/cuda_utils.cpp", "csrc/cuda_utils_kernels.cu"],
202     extra_compile_args={
203         "cxx": CXX_FLAGS,
204         "nvcc": NVCC_FLAGS,
205     },
206 )
207 ext_modules.append(cuda_utils_extension)
208 
209 
210 def get_path(*filepath) -> str:
211     return os.path.join(ROOT_DIR, *filepath)
212 
213 
214 def find_version(filepath: str):
215     """Extract version information from the given filepath.
216 
217     Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
218     """
219     with open(filepath) as fp:
220         version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
221                                   fp.read(), re.M)
222         if version_match:
223             return version_match.group(1)
224         raise RuntimeError("Unable to find version string.")
225 
226 
227 def read_readme() -> str:
228     """Read the README file."""
229     return io.open(get_path("README.md"), "r", encoding="utf-8").read()
230 
231 
232 def get_requirements() -> List[str]:
233     """Get Python package dependencies from requirements.txt."""
234     with open(get_path("requirements.txt")) as f:
235         requirements = f.read().strip().split("\n")
236     return requirements
237 
238 
239 setuptools.setup(
240     name="vllm",
241     version=find_version(get_path("vllm", "__init__.py")),
242     author="vLLM Team",
243     license="Apache 2.0",
244     description=("A high-throughput and memory-efficient inference and "
245                  "serving engine for LLMs"),
246     long_description=read_readme(),
247     long_description_content_type="text/markdown",
248     url="https://github.com/vllm-project/vllm",
249     project_urls={
250         "Homepage": "https://github.com/vllm-project/vllm",
251         "Documentation": "https://vllm.readthedocs.io/en/latest/",
252     },
253     classifiers=[
254         "Programming Language :: Python :: 3.8",
255         "Programming Language :: Python :: 3.9",
256         "Programming Language :: Python :: 3.10",
257         "Programming Language :: Python :: 3.11",
258         "License :: OSI Approved :: Apache Software License",
259         "Topic :: Scientific/Engineering :: Artificial Intelligence",
260     ],
261     packages=setuptools.find_packages(exclude=("benchmarks", "csrc", "docs",
262                                                "examples", "tests")),
263     python_requires=">=3.8",
264     install_requires=get_requirements(),
265     ext_modules=ext_modules,
266     cmdclass={"build_ext": BuildExtension},
267 )
268
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -13,7 +13,7 @@
 ROOT_DIR = os.path.dirname(__file__)
 
 # Supported NVIDIA GPU architectures.
-SUPPORTED_ARCHS = ["7.0", "7.5", "8.0", "8.6", "8.9", "9.0"]
+SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
 
 # Compiler flags.
 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
@@ -49,19 +49,32 @@
     # and executed on the 8.6 or newer architectures. While the PTX code will
     # not give the best performance on the newer architectures, it provides
     # forward compatibility.
-    valid_arch_strs = SUPPORTED_ARCHS + [s + "+PTX" for s in SUPPORTED_ARCHS]
-    arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
-    if arch_list is None:
+    env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
+    if env_arch_list is None:
         return set()
 
     # List are separated by ; or space.
-    arch_list = arch_list.replace(" ", ";").split(";")
-    for arch in arch_list:
-        if arch not in valid_arch_strs:
-            raise ValueError(
-                f"Unsupported CUDA arch ({arch}). "
-                f"Valid CUDA arch strings are: {valid_arch_strs}.")
-    return set(arch_list)
+    torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
+    if not torch_arch_list:
+        return set()
+
+    # Filter out the invalid architectures and print a warning.
+    valid_archs = SUPPORTED_ARCHS.union({s + "+PTX" for s in SUPPORTED_ARCHS})
+    arch_list = torch_arch_list.intersection(valid_archs)
+    # If none of the specified architectures are valid, raise an error.
+    if not arch_list:
+        raise RuntimeError(
+            "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
+            f"variable ({env_arch_list}) is supported. "
+            f"Supported CUDA architectures are: {valid_archs}.")
+    invalid_arch_list = torch_arch_list - valid_archs
+    if invalid_arch_list:
+        warnings.warn(
+            f"Unsupported CUDA architectures ({invalid_arch_list}) are "
+            "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
+            f"({env_arch_list}). Supported CUDA architectures are: "
+            f"{valid_archs}.")
+    return arch_list
 
 
 # First, check the TORCH_CUDA_ARCH_LIST environment variable.
@@ -81,7 +94,7 @@
 if not compute_capabilities:
     # If no GPU is specified nor available, add all supported architectures
     # based on the NVCC CUDA version.
-    compute_capabilities = set(SUPPORTED_ARCHS)
+    compute_capabilities = SUPPORTED_ARCHS.copy()
     if nvcc_cuda_version < Version("11.1"):
         compute_capabilities.remove("8.6")
     if nvcc_cuda_version < Version("11.8"):

{"golden_diff": "diff --git a/setup.py b/setup.py\n--- a/setup.py\n+++ b/setup.py\n@@ -13,7 +13,7 @@\n ROOT_DIR = os.path.dirname(__file__)\n \n # Supported NVIDIA GPU architectures.\n-SUPPORTED_ARCHS = [\"7.0\", \"7.5\", \"8.0\", \"8.6\", \"8.9\", \"9.0\"]\n+SUPPORTED_ARCHS = {\"7.0\", \"7.5\", \"8.0\", \"8.6\", \"8.9\", \"9.0\"}\n \n # Compiler flags.\n CXX_FLAGS = [\"-g\", \"-O2\", \"-std=c++17\"]\n@@ -49,19 +49,32 @@\n     # and executed on the 8.6 or newer architectures. While the PTX code will\n     # not give the best performance on the newer architectures, it provides\n     # forward compatibility.\n-    valid_arch_strs = SUPPORTED_ARCHS + [s + \"+PTX\" for s in SUPPORTED_ARCHS]\n-    arch_list = os.environ.get(\"TORCH_CUDA_ARCH_LIST\", None)\n-    if arch_list is None:\n+    env_arch_list = os.environ.get(\"TORCH_CUDA_ARCH_LIST\", None)\n+    if env_arch_list is None:\n         return set()\n \n     # List are separated by ; or space.\n-    arch_list = arch_list.replace(\" \", \";\").split(\";\")\n-    for arch in arch_list:\n-        if arch not in valid_arch_strs:\n-            raise ValueError(\n-                f\"Unsupported CUDA arch ({arch}). \"\n-                f\"Valid CUDA arch strings are: {valid_arch_strs}.\")\n-    return set(arch_list)\n+    torch_arch_list = set(env_arch_list.replace(\" \", \";\").split(\";\"))\n+    if not torch_arch_list:\n+        return set()\n+\n+    # Filter out the invalid architectures and print a warning.\n+    valid_archs = SUPPORTED_ARCHS.union({s + \"+PTX\" for s in SUPPORTED_ARCHS})\n+    arch_list = torch_arch_list.intersection(valid_archs)\n+    # If none of the specified architectures are valid, raise an error.\n+    if not arch_list:\n+        raise RuntimeError(\n+            \"None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env \"\n+            f\"variable ({env_arch_list}) is supported. \"\n+            f\"Supported CUDA architectures are: {valid_archs}.\")\n+    invalid_arch_list = torch_arch_list - valid_archs\n+    if invalid_arch_list:\n+        warnings.warn(\n+            f\"Unsupported CUDA architectures ({invalid_arch_list}) are \"\n+            \"excluded from the `TORCH_CUDA_ARCH_LIST` env variable \"\n+            f\"({env_arch_list}). Supported CUDA architectures are: \"\n+            f\"{valid_archs}.\")\n+    return arch_list\n \n \n # First, check the TORCH_CUDA_ARCH_LIST environment variable.\n@@ -81,7 +94,7 @@\n if not compute_capabilities:\n     # If no GPU is specified nor available, add all supported architectures\n     # based on the NVCC CUDA version.\n-    compute_capabilities = set(SUPPORTED_ARCHS)\n+    compute_capabilities = SUPPORTED_ARCHS.copy()\n     if nvcc_cuda_version < Version(\"11.1\"):\n         compute_capabilities.remove(\"8.6\")\n     if nvcc_cuda_version < Version(\"11.8\"):\n", "issue": "Installation Error\nWhen I install vLLM, I have pull the docker as the tutorial and install from source. But there are an error encountered, how can I fix this:\r\n`ValueError: Unsupported CUDA arch (5.2). Valid CUDA arch strings are: ['7.0', '7.5', '8.0', '8.6', '8.9', '9.0', '7.0+PTX', '7.5+PTX', '8.0+PTX', '8.6+PTX', '8.9+PTX', '9.0+PTX'].`\n", "before_files": [{"content": "import io\nimport os\nimport re\nimport subprocess\nfrom typing import List, Set\nimport warnings\n\nfrom packaging.version import parse, Version\nimport setuptools\nimport torch\nfrom torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME\n\nROOT_DIR = os.path.dirname(__file__)\n\n# Supported NVIDIA GPU architectures.\nSUPPORTED_ARCHS = [\"7.0\", \"7.5\", \"8.0\", \"8.6\", \"8.9\", \"9.0\"]\n\n# Compiler flags.\nCXX_FLAGS = [\"-g\", \"-O2\", \"-std=c++17\"]\n# TODO(woosuk): Should we use -O3?\nNVCC_FLAGS = [\"-O2\", \"-std=c++17\"]\n\nABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0\nCXX_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\nNVCC_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\n\nif CUDA_HOME is None:\n    raise RuntimeError(\n        \"Cannot find CUDA_HOME. CUDA must be available to build the package.\")\n\n\ndef get_nvcc_cuda_version(cuda_dir: str) -> Version:\n    \"\"\"Get the CUDA version from nvcc.\n\n    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py\n    \"\"\"\n    nvcc_output = subprocess.check_output([cuda_dir + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef get_torch_arch_list() -> Set[str]:\n    # TORCH_CUDA_ARCH_LIST can have one or more architectures,\n    # e.g. \"8.0\" or \"7.5,8.0,8.6+PTX\". Here, the \"8.6+PTX\" option asks the\n    # compiler to additionally include PTX code that can be runtime-compiled\n    # and executed on the 8.6 or newer architectures. While the PTX code will\n    # not give the best performance on the newer architectures, it provides\n    # forward compatibility.\n    valid_arch_strs = SUPPORTED_ARCHS + [s + \"+PTX\" for s in SUPPORTED_ARCHS]\n    arch_list = os.environ.get(\"TORCH_CUDA_ARCH_LIST\", None)\n    if arch_list is None:\n        return set()\n\n    # List are separated by ; or space.\n    arch_list = arch_list.replace(\" \", \";\").split(\";\")\n    for arch in arch_list:\n        if arch not in valid_arch_strs:\n            raise ValueError(\n                f\"Unsupported CUDA arch ({arch}). \"\n                f\"Valid CUDA arch strings are: {valid_arch_strs}.\")\n    return set(arch_list)\n\n\n# First, check the TORCH_CUDA_ARCH_LIST environment variable.\ncompute_capabilities = get_torch_arch_list()\nif not compute_capabilities:\n    # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available\n    # GPUs on the current machine.\n    device_count = torch.cuda.device_count()\n    for i in range(device_count):\n        major, minor = torch.cuda.get_device_capability(i)\n        if major < 7:\n            raise RuntimeError(\n                \"GPUs with compute capability below 7.0 are not supported.\")\n        compute_capabilities.add(f\"{major}.{minor}\")\n\nnvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)\nif not compute_capabilities:\n    # If no GPU is specified nor available, add all supported architectures\n    # based on the NVCC CUDA version.\n    compute_capabilities = set(SUPPORTED_ARCHS)\n    if nvcc_cuda_version < Version(\"11.1\"):\n        compute_capabilities.remove(\"8.6\")\n    if nvcc_cuda_version < Version(\"11.8\"):\n        compute_capabilities.remove(\"8.9\")\n        compute_capabilities.remove(\"9.0\")\n\n# Validate the NVCC CUDA version.\nif nvcc_cuda_version < Version(\"11.0\"):\n    raise RuntimeError(\"CUDA 11.0 or higher is required to build the package.\")\nif nvcc_cuda_version < Version(\"11.1\"):\n    if any(cc.startswith(\"8.6\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.1 or higher is required for compute capability 8.6.\")\nif nvcc_cuda_version < Version(\"11.8\"):\n    if any(cc.startswith(\"8.9\") for cc in compute_capabilities):\n        # CUDA 11.8 is required to generate the code targeting compute capability 8.9.\n        # However, GPUs with compute capability 8.9 can also run the code generated by\n        # the previous versions of CUDA 11 and targeting compute capability 8.0.\n        # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0\n        # instead of 8.9.\n        warnings.warn(\n            \"CUDA 11.8 or higher is required for compute capability 8.9. \"\n            \"Targeting compute capability 8.0 instead.\")\n        compute_capabilities = set(cc for cc in compute_capabilities\n                                   if not cc.startswith(\"8.9\"))\n        compute_capabilities.add(\"8.0+PTX\")\n    if any(cc.startswith(\"9.0\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.8 or higher is required for compute capability 9.0.\")\n\n# Add target compute capabilities to NVCC flags.\nfor capability in compute_capabilities:\n    num = capability[0] + capability[2]\n    NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=sm_{num}\"]\n    if capability.endswith(\"+PTX\"):\n        NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=compute_{num}\"]\n\n# Use NVCC threads to parallelize the build.\nif nvcc_cuda_version >= Version(\"11.2\"):\n    num_threads = min(os.cpu_count(), 8)\n    NVCC_FLAGS += [\"--threads\", str(num_threads)]\n\next_modules = []\n\n# Cache operations.\ncache_extension = CUDAExtension(\n    name=\"vllm.cache_ops\",\n    sources=[\"csrc/cache.cpp\", \"csrc/cache_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cache_extension)\n\n# Attention kernels.\nattention_extension = CUDAExtension(\n    name=\"vllm.attention_ops\",\n    sources=[\"csrc/attention.cpp\", \"csrc/attention/attention_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(attention_extension)\n\n# Positional encoding kernels.\npositional_encoding_extension = CUDAExtension(\n    name=\"vllm.pos_encoding_ops\",\n    sources=[\"csrc/pos_encoding.cpp\", \"csrc/pos_encoding_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(positional_encoding_extension)\n\n# Layer normalization kernels.\nlayernorm_extension = CUDAExtension(\n    name=\"vllm.layernorm_ops\",\n    sources=[\"csrc/layernorm.cpp\", \"csrc/layernorm_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(layernorm_extension)\n\n# Activation kernels.\nactivation_extension = CUDAExtension(\n    name=\"vllm.activation_ops\",\n    sources=[\"csrc/activation.cpp\", \"csrc/activation_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(activation_extension)\n\n# Quantization kernels.\nquantization_extension = CUDAExtension(\n    name=\"vllm.quantization_ops\",\n    sources=[\n        \"csrc/quantization.cpp\",\n        \"csrc/quantization/awq/gemm_kernels.cu\",\n    ],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(quantization_extension)\n\n# Misc. CUDA utils.\ncuda_utils_extension = CUDAExtension(\n    name=\"vllm.cuda_utils\",\n    sources=[\"csrc/cuda_utils.cpp\", \"csrc/cuda_utils_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cuda_utils_extension)\n\n\ndef get_path(*filepath) -> str:\n    return os.path.join(ROOT_DIR, *filepath)\n\n\ndef find_version(filepath: str):\n    \"\"\"Extract version information from the given filepath.\n\n    Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py\n    \"\"\"\n    with open(filepath) as fp:\n        version_match = re.search(r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\",\n                                  fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\ndef read_readme() -> str:\n    \"\"\"Read the README file.\"\"\"\n    return io.open(get_path(\"README.md\"), \"r\", encoding=\"utf-8\").read()\n\n\ndef get_requirements() -> List[str]:\n    \"\"\"Get Python package dependencies from requirements.txt.\"\"\"\n    with open(get_path(\"requirements.txt\")) as f:\n        requirements = f.read().strip().split(\"\\n\")\n    return requirements\n\n\nsetuptools.setup(\n    name=\"vllm\",\n    version=find_version(get_path(\"vllm\", \"__init__.py\")),\n    author=\"vLLM Team\",\n    license=\"Apache 2.0\",\n    description=(\"A high-throughput and memory-efficient inference and \"\n                 \"serving engine for LLMs\"),\n    long_description=read_readme(),\n    long_description_content_type=\"text/markdown\",\n    url=\"https://github.com/vllm-project/vllm\",\n    project_urls={\n        \"Homepage\": \"https://github.com/vllm-project/vllm\",\n        \"Documentation\": \"https://vllm.readthedocs.io/en/latest/\",\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.8\",\n        \"Programming Language :: Python :: 3.9\",\n        \"Programming Language :: Python :: 3.10\",\n        \"Programming Language :: Python :: 3.11\",\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n    ],\n    packages=setuptools.find_packages(exclude=(\"benchmarks\", \"csrc\", \"docs\",\n                                               \"examples\", \"tests\")),\n    python_requires=\">=3.8\",\n    install_requires=get_requirements(),\n    ext_modules=ext_modules,\n    cmdclass={\"build_ext\": BuildExtension},\n)\n", "path": "setup.py"}], "after_files": [{"content": "import io\nimport os\nimport re\nimport subprocess\nfrom typing import List, Set\nimport warnings\n\nfrom packaging.version import parse, Version\nimport setuptools\nimport torch\nfrom torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME\n\nROOT_DIR = os.path.dirname(__file__)\n\n# Supported NVIDIA GPU architectures.\nSUPPORTED_ARCHS = {\"7.0\", \"7.5\", \"8.0\", \"8.6\", \"8.9\", \"9.0\"}\n\n# Compiler flags.\nCXX_FLAGS = [\"-g\", \"-O2\", \"-std=c++17\"]\n# TODO(woosuk): Should we use -O3?\nNVCC_FLAGS = [\"-O2\", \"-std=c++17\"]\n\nABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0\nCXX_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\nNVCC_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\n\nif CUDA_HOME is None:\n    raise RuntimeError(\n        \"Cannot find CUDA_HOME. CUDA must be available to build the package.\")\n\n\ndef get_nvcc_cuda_version(cuda_dir: str) -> Version:\n    \"\"\"Get the CUDA version from nvcc.\n\n    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py\n    \"\"\"\n    nvcc_output = subprocess.check_output([cuda_dir + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef get_torch_arch_list() -> Set[str]:\n    # TORCH_CUDA_ARCH_LIST can have one or more architectures,\n    # e.g. \"8.0\" or \"7.5,8.0,8.6+PTX\". Here, the \"8.6+PTX\" option asks the\n    # compiler to additionally include PTX code that can be runtime-compiled\n    # and executed on the 8.6 or newer architectures. While the PTX code will\n    # not give the best performance on the newer architectures, it provides\n    # forward compatibility.\n    env_arch_list = os.environ.get(\"TORCH_CUDA_ARCH_LIST\", None)\n    if env_arch_list is None:\n        return set()\n\n    # List are separated by ; or space.\n    torch_arch_list = set(env_arch_list.replace(\" \", \";\").split(\";\"))\n    if not torch_arch_list:\n        return set()\n\n    # Filter out the invalid architectures and print a warning.\n    valid_archs = SUPPORTED_ARCHS.union({s + \"+PTX\" for s in SUPPORTED_ARCHS})\n    arch_list = torch_arch_list.intersection(valid_archs)\n    # If none of the specified architectures are valid, raise an error.\n    if not arch_list:\n        raise RuntimeError(\n            \"None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env \"\n            f\"variable ({env_arch_list}) is supported. \"\n            f\"Supported CUDA architectures are: {valid_archs}.\")\n    invalid_arch_list = torch_arch_list - valid_archs\n    if invalid_arch_list:\n        warnings.warn(\n            f\"Unsupported CUDA architectures ({invalid_arch_list}) are \"\n            \"excluded from the `TORCH_CUDA_ARCH_LIST` env variable \"\n            f\"({env_arch_list}). Supported CUDA architectures are: \"\n            f\"{valid_archs}.\")\n    return arch_list\n\n\n# First, check the TORCH_CUDA_ARCH_LIST environment variable.\ncompute_capabilities = get_torch_arch_list()\nif not compute_capabilities:\n    # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available\n    # GPUs on the current machine.\n    device_count = torch.cuda.device_count()\n    for i in range(device_count):\n        major, minor = torch.cuda.get_device_capability(i)\n        if major < 7:\n            raise RuntimeError(\n                \"GPUs with compute capability below 7.0 are not supported.\")\n        compute_capabilities.add(f\"{major}.{minor}\")\n\nnvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)\nif not compute_capabilities:\n    # If no GPU is specified nor available, add all supported architectures\n    # based on the NVCC CUDA version.\n    compute_capabilities = SUPPORTED_ARCHS.copy()\n    if nvcc_cuda_version < Version(\"11.1\"):\n        compute_capabilities.remove(\"8.6\")\n    if nvcc_cuda_version < Version(\"11.8\"):\n        compute_capabilities.remove(\"8.9\")\n        compute_capabilities.remove(\"9.0\")\n\n# Validate the NVCC CUDA version.\nif nvcc_cuda_version < Version(\"11.0\"):\n    raise RuntimeError(\"CUDA 11.0 or higher is required to build the package.\")\nif nvcc_cuda_version < Version(\"11.1\"):\n    if any(cc.startswith(\"8.6\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.1 or higher is required for compute capability 8.6.\")\nif nvcc_cuda_version < Version(\"11.8\"):\n    if any(cc.startswith(\"8.9\") for cc in compute_capabilities):\n        # CUDA 11.8 is required to generate the code targeting compute capability 8.9.\n        # However, GPUs with compute capability 8.9 can also run the code generated by\n        # the previous versions of CUDA 11 and targeting compute capability 8.0.\n        # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0\n        # instead of 8.9.\n        warnings.warn(\n            \"CUDA 11.8 or higher is required for compute capability 8.9. \"\n            \"Targeting compute capability 8.0 instead.\")\n        compute_capabilities = set(cc for cc in compute_capabilities\n                                   if not cc.startswith(\"8.9\"))\n        compute_capabilities.add(\"8.0+PTX\")\n    if any(cc.startswith(\"9.0\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.8 or higher is required for compute capability 9.0.\")\n\n# Add target compute capabilities to NVCC flags.\nfor capability in compute_capabilities:\n    num = capability[0] + capability[2]\n    NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=sm_{num}\"]\n    if capability.endswith(\"+PTX\"):\n        NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=compute_{num}\"]\n\n# Use NVCC threads to parallelize the build.\nif nvcc_cuda_version >= Version(\"11.2\"):\n    num_threads = min(os.cpu_count(), 8)\n    NVCC_FLAGS += [\"--threads\", str(num_threads)]\n\next_modules = []\n\n# Cache operations.\ncache_extension = CUDAExtension(\n    name=\"vllm.cache_ops\",\n    sources=[\"csrc/cache.cpp\", \"csrc/cache_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cache_extension)\n\n# Attention kernels.\nattention_extension = CUDAExtension(\n    name=\"vllm.attention_ops\",\n    sources=[\"csrc/attention.cpp\", \"csrc/attention/attention_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(attention_extension)\n\n# Positional encoding kernels.\npositional_encoding_extension = CUDAExtension(\n    name=\"vllm.pos_encoding_ops\",\n    sources=[\"csrc/pos_encoding.cpp\", \"csrc/pos_encoding_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(positional_encoding_extension)\n\n# Layer normalization kernels.\nlayernorm_extension = CUDAExtension(\n    name=\"vllm.layernorm_ops\",\n    sources=[\"csrc/layernorm.cpp\", \"csrc/layernorm_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(layernorm_extension)\n\n# Activation kernels.\nactivation_extension = CUDAExtension(\n    name=\"vllm.activation_ops\",\n    sources=[\"csrc/activation.cpp\", \"csrc/activation_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(activation_extension)\n\n# Quantization kernels.\nquantization_extension = CUDAExtension(\n    name=\"vllm.quantization_ops\",\n    sources=[\n        \"csrc/quantization.cpp\",\n        \"csrc/quantization/awq/gemm_kernels.cu\",\n    ],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(quantization_extension)\n\n# Misc. CUDA utils.\ncuda_utils_extension = CUDAExtension(\n    name=\"vllm.cuda_utils\",\n    sources=[\"csrc/cuda_utils.cpp\", \"csrc/cuda_utils_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cuda_utils_extension)\n\n\ndef get_path(*filepath) -> str:\n    return os.path.join(ROOT_DIR, *filepath)\n\n\ndef find_version(filepath: str):\n    \"\"\"Extract version information from the given filepath.\n\n    Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py\n    \"\"\"\n    with open(filepath) as fp:\n        version_match = re.search(r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\",\n                                  fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\ndef read_readme() -> str:\n    \"\"\"Read the README file.\"\"\"\n    return io.open(get_path(\"README.md\"), \"r\", encoding=\"utf-8\").read()\n\n\ndef get_requirements() -> List[str]:\n    \"\"\"Get Python package dependencies from requirements.txt.\"\"\"\n    with open(get_path(\"requirements.txt\")) as f:\n        requirements = f.read().strip().split(\"\\n\")\n    return requirements\n\n\nsetuptools.setup(\n    name=\"vllm\",\n    version=find_version(get_path(\"vllm\", \"__init__.py\")),\n    author=\"vLLM Team\",\n    license=\"Apache 2.0\",\n    description=(\"A high-throughput and memory-efficient inference and \"\n                 \"serving engine for LLMs\"),\n    long_description=read_readme(),\n    long_description_content_type=\"text/markdown\",\n    url=\"https://github.com/vllm-project/vllm\",\n    project_urls={\n        \"Homepage\": \"https://github.com/vllm-project/vllm\",\n        \"Documentation\": \"https://vllm.readthedocs.io/en/latest/\",\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.8\",\n        \"Programming Language :: Python :: 3.9\",\n        \"Programming Language :: Python :: 3.10\",\n        \"Programming Language :: Python :: 3.11\",\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n    ],\n    packages=setuptools.find_packages(exclude=(\"benchmarks\", \"csrc\", \"docs\",\n                                               \"examples\", \"tests\")),\n    python_requires=\">=3.8\",\n    install_requires=get_requirements(),\n    ext_modules=ext_modules,\n    cmdclass={\"build_ext\": BuildExtension},\n)\n", "path": "setup.py"}]}