We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Benchmarking script for openai chat completion api are not supported
When running vllm with openai chat apis, the benchmarking script will fail as it asserts the backend API of `assert api_url.endswith("v1/completions")`.

```
python benchmark_serving.py --backend openai --model mistralai/Mistral-7B-v0.1 --dataset ShareGPT_V3_unfiltered_cleaned_split.json --save-result
```

The logs are as follows:
```
Namespace(backend='openai', version='N/A', base_url=None, host='localhost', port=8000, endpoint='/generate', dataset='ShareGPT_V3_unfiltered_cleaned_split.json', model='mistralai/Mistral-7B-v0.1', tokenizer=None, best_of=1, use_beam_search=False, num_prompts=1000, request_rate=inf, seed=0, trust_remote_code=False, disable_tqdm=False, save_result=True)
  0%|                                                      | 0/1000 [00:00<?, ?it/s]Traffic request rate: inf
Traceback (most recent call last):
  File "/home/chenw/vllm/benchmarks/benchmark_serving.py", line 387, in <module>
    main(args)
  File "/home/chenw/vllm/benchmarks/benchmark_serving.py", line 259, in main
    benchmark_result = asyncio.run(
  File "/home/chenw/miniconda3/envs/myenv/lib/python3.9/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/chenw/miniconda3/envs/myenv/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
    return future.result()
  File "/home/chenw/vllm/benchmarks/benchmark_serving.py", line 195, in benchmark
    outputs = await asyncio.gather(*tasks)
  File "/home/chenw/vllm/benchmarks/backend_request_func.py", line 223, in async_request_openai_completions
    assert api_url.endswith("v1/completions")
AssertionError
  0%|                                                      | 0/1000 [00:00<?, ?it/s]
```

The `backend_request_func.py` should not only allow chat apis like: `assert api_url.endswith("v1/chat/completions")`.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `benchmarks/backend_request_func.py`
Content:
```
1 import json
2 import os
3 import time
4 from dataclasses import dataclass
5 from typing import Optional
6 
7 import aiohttp
8 from tqdm.asyncio import tqdm
9 
10 AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
11 
12 
13 @dataclass
14 class RequestFuncInput:
15     prompt: str
16     api_url: str
17     prompt_len: int
18     output_len: int
19     model: str
20     best_of: int = 1
21     use_beam_search: bool = False
22 
23 
24 @dataclass
25 class RequestFuncOutput:
26     generated_text: str = ""
27     success: bool = False
28     latency: float = 0
29     ttft: float = 0
30     prompt_len: int = 0
31 
32 
33 async def async_request_tgi(
34     request_func_input: RequestFuncInput,
35     pbar: Optional[tqdm] = None,
36 ) -> RequestFuncOutput:
37     api_url = request_func_input.api_url
38     assert api_url.endswith("generate_stream")
39 
40     async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
41         assert not request_func_input.use_beam_search
42         params = {
43             "best_of": request_func_input.best_of,
44             "max_new_tokens": request_func_input.output_len,
45             "do_sample": True,
46             "temperature": 0.01,  # TGI does not accept 0.0 temperature.
47             "top_p": 0.99,  # TGI does not accept 1.0 top_p.
48         }
49         payload = {
50             "inputs": request_func_input.prompt,
51             "parameters": params,
52         }
53         output = RequestFuncOutput()
54         output.prompt_len = request_func_input.prompt_len
55 
56         ttft = 0
57         st = time.perf_counter()
58         try:
59             async with session.post(url=api_url, json=payload) as response:
60                 if response.status == 200:
61                     async for data in response.content.iter_any():
62                         if ttft == 0:
63                             ttft = time.perf_counter() - st
64                             output.ttft = ttft
65                     output.latency = time.perf_counter() - st
66 
67                     body = data.decode("utf-8").lstrip("data:")
68                     output.generated_text = json.loads(body)["generated_text"]
69                     output.success = True
70                 else:
71                     output.success = False
72         except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
73             output.success = False
74 
75         if pbar:
76             pbar.update(1)
77         return output
78 
79 
80 async def async_request_vllm(
81     request_func_input: RequestFuncInput,
82     pbar: Optional[tqdm] = None,
83 ) -> RequestFuncOutput:
84     api_url = request_func_input.api_url
85     assert api_url.endswith("generate")
86 
87     async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
88         payload = {
89             "prompt": request_func_input.prompt,
90             "n": 1,
91             "best_of": request_func_input.best_of,
92             "use_beam_search": request_func_input.use_beam_search,
93             "temperature": 0.0 if request_func_input.use_beam_search else 1.0,
94             "top_p": 1.0,
95             "max_tokens": request_func_input.output_len,
96             "ignore_eos": True,
97             "stream": True,
98         }
99         output = RequestFuncOutput()
100         output.prompt_len = request_func_input.prompt_len
101 
102         ttft = 0
103         st = time.perf_counter()
104         try:
105             async with session.post(url=api_url, json=payload) as response:
106                 if response.status == 200:
107                     async for data in response.content.iter_any():
108                         if ttft == 0:
109                             ttft = time.perf_counter() - st
110                             output.ttft = ttft
111                     output.latency = time.perf_counter() - st
112 
113                     # When streaming, '\0' is appended to the end of the response.
114                     body = data.decode("utf-8").strip("\0")
115                     output.generated_text = json.loads(
116                         body)["text"][0][len(request_func_input.prompt):]
117                     output.success = True
118 
119                 else:
120                     output.success = False
121         except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
122             output.success = False
123 
124         if pbar:
125             pbar.update(1)
126         return output
127 
128 
129 async def async_request_trt_llm(
130     request_func_input: RequestFuncInput,
131     pbar: Optional[tqdm] = None,
132 ) -> RequestFuncOutput:
133     api_url = request_func_input.api_url
134     assert api_url.endswith("generate_stream")
135 
136     async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
137         assert not request_func_input.use_beam_search
138         assert request_func_input.best_of == 1
139         payload = {
140             "accumulate_tokens": True,
141             "text_input": request_func_input.prompt,
142             "temperature": 0.0,
143             "top_p": 1.0,
144             "max_tokens": request_func_input.output_len,
145             "stream": True,
146         }
147         output = RequestFuncOutput()
148         output.prompt_len = request_func_input.prompt_len
149         ttft = 0
150 
151         st = time.perf_counter()
152         try:
153             async with session.post(url=api_url, json=payload) as resp:
154                 if resp.status == 200:
155                     async for data in resp.content.iter_any():
156                         if ttft == 0:
157                             ttft = time.perf_counter() - st
158                             output.ttft = ttft
159                     output.latency = time.perf_counter() - st
160 
161                     body = data.decode("utf-8").lstrip("data:")
162                     output.generated_text = json.loads(body)["text_output"]
163                     output.success = True
164 
165                 else:
166                     output.success = False
167         except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
168             output.success = False
169 
170         if pbar:
171             pbar.update(1)
172         return output
173 
174 
175 async def async_request_deepspeed_mii(
176     request_func_input: RequestFuncInput,
177     pbar: Optional[tqdm] = None,
178 ) -> RequestFuncOutput:
179     async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
180         assert request_func_input.best_of == 1
181         assert not request_func_input.use_beam_search
182 
183         payload = {
184             "prompts": request_func_input.prompt,
185             "max_new_tokens": request_func_input.output_len,
186             "ignore_eos": True,
187             "do_sample": True,
188             "temperature":
189             0.01,  # deepspeed-mii does not accept 0.0 temperature.
190             "top_p": 1.0,
191         }
192         output = RequestFuncOutput()
193         output.prompt_len = request_func_input.prompt_len
194 
195         # DeepSpeed-MII doesn't support streaming as of Jan 28 2024, will use 0 as placeholder.
196         # https://github.com/microsoft/DeepSpeed-MII/pull/311
197         output.ttft = 0
198 
199         st = time.perf_counter()
200         try:
201             async with session.post(url=request_func_input.api_url,
202                                     json=payload) as resp:
203                 if resp.status == 200:
204                     parsed_resp = await resp.json()
205                     output.latency = time.perf_counter() - st
206                     output.generated_text = parsed_resp[0]["generated_text"]
207                     output.success = True
208                 else:
209                     output.success = False
210         except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
211             output.success = False
212 
213         if pbar:
214             pbar.update(1)
215         return output
216 
217 
218 async def async_request_openai_completions(
219     request_func_input: RequestFuncInput,
220     pbar: Optional[tqdm] = None,
221 ) -> RequestFuncOutput:
222     api_url = request_func_input.api_url
223     assert api_url.endswith("v1/completions")
224 
225     async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
226         assert not request_func_input.use_beam_search
227         payload = {
228             "model": request_func_input.model,
229             "prompt": request_func_input.prompt,
230             "temperature": 0.0,
231             "best_of": request_func_input.best_of,
232             "max_tokens": request_func_input.output_len,
233             "stream": True,
234         }
235         headers = {
236             "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
237         }
238 
239         output = RequestFuncOutput()
240         output.prompt_len = request_func_input.prompt_len
241 
242         generated_text = ""
243         ttft = 0
244         st = time.perf_counter()
245         try:
246             async with session.post(url=api_url, json=payload,
247                                     headers=headers) as response:
248                 if response.status == 200:
249                     async for chunk in response.content:
250                         if ttft == 0:
251                             ttft = time.perf_counter() - st
252                             output.ttft = ttft
253 
254                         chunk = chunk.strip()
255                         if not chunk:
256                             continue
257 
258                         chunk = chunk.decode("utf-8").lstrip("data: ")
259                         if chunk == "[DONE]":
260                             latency = time.perf_counter() - st
261                         else:
262                             body = json.loads(chunk)
263                             generated_text += body["choices"][0]["text"]
264 
265                     output.generated_text = generated_text
266                     output.success = True
267                     output.latency = latency
268                 else:
269                     output.success = False
270         except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
271             output.success = False
272 
273     if pbar:
274         pbar.update(1)
275     return output
276 
277 
278 ASYNC_REQUEST_FUNCS = {
279     "tgi": async_request_tgi,
280     "vllm": async_request_vllm,
281     "deepspeed-mii": async_request_deepspeed_mii,
282     "openai": async_request_openai_completions,
283     "tensorrt-llm": async_request_trt_llm,
284 }
285
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py
--- a/benchmarks/backend_request_func.py
+++ b/benchmarks/backend_request_func.py
@@ -275,10 +275,80 @@
     return output
 
 
+async def async_request_openai_chat_completions(
+    request_func_input: RequestFuncInput,
+    pbar: Optional[tqdm] = None,
+) -> RequestFuncOutput:
+    api_url = request_func_input.api_url
+    assert api_url.endswith(
+        "v1/chat/completions"
+    ), "OpenAI Chat API URL must end with 'v1/chat/completions'."
+
+    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
+        assert not request_func_input.use_beam_search
+        payload = {
+            "model": request_func_input.model,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": request_func_input.prompt,
+                },
+            ],
+            "temperature": 0.0,
+            "max_tokens": request_func_input.output_len,
+            "stream": True,
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
+        }
+
+        output = RequestFuncOutput()
+        output.prompt_len = request_func_input.prompt_len
+
+        generated_text = ""
+        ttft = 0
+        st = time.perf_counter()
+        try:
+            async with session.post(url=api_url, json=payload,
+                                    headers=headers) as response:
+                if response.status == 200:
+                    async for chunk in response.content:
+                        if ttft == 0:
+                            ttft = time.perf_counter() - st
+                            output.ttft = ttft
+
+                        chunk = chunk.strip()
+                        if not chunk:
+                            continue
+
+                        chunk = chunk.decode("utf-8").lstrip("data: ")
+                        if chunk == "[DONE]":
+                            latency = time.perf_counter() - st
+                        else:
+                            body = json.loads(chunk)
+                            if "content" in body["choices"][0]["delta"]:
+                                generated_text += body["choices"][0]["delta"][
+                                    "content"]
+
+                    output.generated_text = generated_text
+                    output.success = True
+                    output.latency = latency
+                else:
+                    output.success = False
+        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
+            output.success = False
+
+    if pbar:
+        pbar.update(1)
+    return output
+
+
 ASYNC_REQUEST_FUNCS = {
     "tgi": async_request_tgi,
     "vllm": async_request_vllm,
     "deepspeed-mii": async_request_deepspeed_mii,
     "openai": async_request_openai_completions,
+    "openai-chat": async_request_openai_chat_completions,
     "tensorrt-llm": async_request_trt_llm,
 }

{"golden_diff": "diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py\n--- a/benchmarks/backend_request_func.py\n+++ b/benchmarks/backend_request_func.py\n@@ -275,10 +275,80 @@\n     return output\n \n \n+async def async_request_openai_chat_completions(\n+    request_func_input: RequestFuncInput,\n+    pbar: Optional[tqdm] = None,\n+) -> RequestFuncOutput:\n+    api_url = request_func_input.api_url\n+    assert api_url.endswith(\n+        \"v1/chat/completions\"\n+    ), \"OpenAI Chat API URL must end with 'v1/chat/completions'.\"\n+\n+    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n+        assert not request_func_input.use_beam_search\n+        payload = {\n+            \"model\": request_func_input.model,\n+            \"messages\": [\n+                {\n+                    \"role\": \"user\",\n+                    \"content\": request_func_input.prompt,\n+                },\n+            ],\n+            \"temperature\": 0.0,\n+            \"max_tokens\": request_func_input.output_len,\n+            \"stream\": True,\n+        }\n+        headers = {\n+            \"Content-Type\": \"application/json\",\n+            \"Authorization\": f\"Bearer {os.environ.get('OPENAI_API_KEY')}\"\n+        }\n+\n+        output = RequestFuncOutput()\n+        output.prompt_len = request_func_input.prompt_len\n+\n+        generated_text = \"\"\n+        ttft = 0\n+        st = time.perf_counter()\n+        try:\n+            async with session.post(url=api_url, json=payload,\n+                                    headers=headers) as response:\n+                if response.status == 200:\n+                    async for chunk in response.content:\n+                        if ttft == 0:\n+                            ttft = time.perf_counter() - st\n+                            output.ttft = ttft\n+\n+                        chunk = chunk.strip()\n+                        if not chunk:\n+                            continue\n+\n+                        chunk = chunk.decode(\"utf-8\").lstrip(\"data: \")\n+                        if chunk == \"[DONE]\":\n+                            latency = time.perf_counter() - st\n+                        else:\n+                            body = json.loads(chunk)\n+                            if \"content\" in body[\"choices\"][0][\"delta\"]:\n+                                generated_text += body[\"choices\"][0][\"delta\"][\n+                                    \"content\"]\n+\n+                    output.generated_text = generated_text\n+                    output.success = True\n+                    output.latency = latency\n+                else:\n+                    output.success = False\n+        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n+            output.success = False\n+\n+    if pbar:\n+        pbar.update(1)\n+    return output\n+\n+\n ASYNC_REQUEST_FUNCS = {\n     \"tgi\": async_request_tgi,\n     \"vllm\": async_request_vllm,\n     \"deepspeed-mii\": async_request_deepspeed_mii,\n     \"openai\": async_request_openai_completions,\n+    \"openai-chat\": async_request_openai_chat_completions,\n     \"tensorrt-llm\": async_request_trt_llm,\n }\n", "issue": "Benchmarking script for openai chat completion api are not supported\nWhen running vllm with openai chat apis, the benchmarking script will fail as it asserts the backend API of `assert api_url.endswith(\"v1/completions\")`.\r\n\r\n```\r\npython benchmark_serving.py --backend openai --model mistralai/Mistral-7B-v0.1 --dataset ShareGPT_V3_unfiltered_cleaned_split.json --save-result\r\n```\r\n\r\nThe logs are as follows:\r\n```\r\nNamespace(backend='openai', version='N/A', base_url=None, host='localhost', port=8000, endpoint='/generate', dataset='ShareGPT_V3_unfiltered_cleaned_split.json', model='mistralai/Mistral-7B-v0.1', tokenizer=None, best_of=1, use_beam_search=False, num_prompts=1000, request_rate=inf, seed=0, trust_remote_code=False, disable_tqdm=False, save_result=True)\r\n  0%|                                                      | 0/1000 [00:00<?, ?it/s]Traffic request rate: inf\r\nTraceback (most recent call last):\r\n  File \"/home/chenw/vllm/benchmarks/benchmark_serving.py\", line 387, in <module>\r\n    main(args)\r\n  File \"/home/chenw/vllm/benchmarks/benchmark_serving.py\", line 259, in main\r\n    benchmark_result = asyncio.run(\r\n  File \"/home/chenw/miniconda3/envs/myenv/lib/python3.9/asyncio/runners.py\", line 44, in run\r\n    return loop.run_until_complete(main)\r\n  File \"/home/chenw/miniconda3/envs/myenv/lib/python3.9/asyncio/base_events.py\", line 647, in run_until_complete\r\n    return future.result()\r\n  File \"/home/chenw/vllm/benchmarks/benchmark_serving.py\", line 195, in benchmark\r\n    outputs = await asyncio.gather(*tasks)\r\n  File \"/home/chenw/vllm/benchmarks/backend_request_func.py\", line 223, in async_request_openai_completions\r\n    assert api_url.endswith(\"v1/completions\")\r\nAssertionError\r\n  0%|                                                      | 0/1000 [00:00<?, ?it/s]\r\n```\r\n\r\nThe `backend_request_func.py` should not only allow chat apis like: `assert api_url.endswith(\"v1/chat/completions\")`.\r\n\n", "before_files": [{"content": "import json\nimport os\nimport time\nfrom dataclasses import dataclass\nfrom typing import Optional\n\nimport aiohttp\nfrom tqdm.asyncio import tqdm\n\nAIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)\n\n\n@dataclass\nclass RequestFuncInput:\n    prompt: str\n    api_url: str\n    prompt_len: int\n    output_len: int\n    model: str\n    best_of: int = 1\n    use_beam_search: bool = False\n\n\n@dataclass\nclass RequestFuncOutput:\n    generated_text: str = \"\"\n    success: bool = False\n    latency: float = 0\n    ttft: float = 0\n    prompt_len: int = 0\n\n\nasync def async_request_tgi(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"generate_stream\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert not request_func_input.use_beam_search\n        params = {\n            \"best_of\": request_func_input.best_of,\n            \"max_new_tokens\": request_func_input.output_len,\n            \"do_sample\": True,\n            \"temperature\": 0.01,  # TGI does not accept 0.0 temperature.\n            \"top_p\": 0.99,  # TGI does not accept 1.0 top_p.\n        }\n        payload = {\n            \"inputs\": request_func_input.prompt,\n            \"parameters\": params,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        ttft = 0\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload) as response:\n                if response.status == 200:\n                    async for data in response.content.iter_any():\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n                    output.latency = time.perf_counter() - st\n\n                    body = data.decode(\"utf-8\").lstrip(\"data:\")\n                    output.generated_text = json.loads(body)[\"generated_text\"]\n                    output.success = True\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_vllm(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"generate\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        payload = {\n            \"prompt\": request_func_input.prompt,\n            \"n\": 1,\n            \"best_of\": request_func_input.best_of,\n            \"use_beam_search\": request_func_input.use_beam_search,\n            \"temperature\": 0.0 if request_func_input.use_beam_search else 1.0,\n            \"top_p\": 1.0,\n            \"max_tokens\": request_func_input.output_len,\n            \"ignore_eos\": True,\n            \"stream\": True,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        ttft = 0\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload) as response:\n                if response.status == 200:\n                    async for data in response.content.iter_any():\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n                    output.latency = time.perf_counter() - st\n\n                    # When streaming, '\\0' is appended to the end of the response.\n                    body = data.decode(\"utf-8\").strip(\"\\0\")\n                    output.generated_text = json.loads(\n                        body)[\"text\"][0][len(request_func_input.prompt):]\n                    output.success = True\n\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_trt_llm(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"generate_stream\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert not request_func_input.use_beam_search\n        assert request_func_input.best_of == 1\n        payload = {\n            \"accumulate_tokens\": True,\n            \"text_input\": request_func_input.prompt,\n            \"temperature\": 0.0,\n            \"top_p\": 1.0,\n            \"max_tokens\": request_func_input.output_len,\n            \"stream\": True,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n        ttft = 0\n\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload) as resp:\n                if resp.status == 200:\n                    async for data in resp.content.iter_any():\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n                    output.latency = time.perf_counter() - st\n\n                    body = data.decode(\"utf-8\").lstrip(\"data:\")\n                    output.generated_text = json.loads(body)[\"text_output\"]\n                    output.success = True\n\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_deepspeed_mii(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert request_func_input.best_of == 1\n        assert not request_func_input.use_beam_search\n\n        payload = {\n            \"prompts\": request_func_input.prompt,\n            \"max_new_tokens\": request_func_input.output_len,\n            \"ignore_eos\": True,\n            \"do_sample\": True,\n            \"temperature\":\n            0.01,  # deepspeed-mii does not accept 0.0 temperature.\n            \"top_p\": 1.0,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        # DeepSpeed-MII doesn't support streaming as of Jan 28 2024, will use 0 as placeholder.\n        # https://github.com/microsoft/DeepSpeed-MII/pull/311\n        output.ttft = 0\n\n        st = time.perf_counter()\n        try:\n            async with session.post(url=request_func_input.api_url,\n                                    json=payload) as resp:\n                if resp.status == 200:\n                    parsed_resp = await resp.json()\n                    output.latency = time.perf_counter() - st\n                    output.generated_text = parsed_resp[0][\"generated_text\"]\n                    output.success = True\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_openai_completions(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"v1/completions\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert not request_func_input.use_beam_search\n        payload = {\n            \"model\": request_func_input.model,\n            \"prompt\": request_func_input.prompt,\n            \"temperature\": 0.0,\n            \"best_of\": request_func_input.best_of,\n            \"max_tokens\": request_func_input.output_len,\n            \"stream\": True,\n        }\n        headers = {\n            \"Authorization\": f\"Bearer {os.environ.get('OPENAI_API_KEY')}\"\n        }\n\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        generated_text = \"\"\n        ttft = 0\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload,\n                                    headers=headers) as response:\n                if response.status == 200:\n                    async for chunk in response.content:\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n\n                        chunk = chunk.strip()\n                        if not chunk:\n                            continue\n\n                        chunk = chunk.decode(\"utf-8\").lstrip(\"data: \")\n                        if chunk == \"[DONE]\":\n                            latency = time.perf_counter() - st\n                        else:\n                            body = json.loads(chunk)\n                            generated_text += body[\"choices\"][0][\"text\"]\n\n                    output.generated_text = generated_text\n                    output.success = True\n                    output.latency = latency\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n    if pbar:\n        pbar.update(1)\n    return output\n\n\nASYNC_REQUEST_FUNCS = {\n    \"tgi\": async_request_tgi,\n    \"vllm\": async_request_vllm,\n    \"deepspeed-mii\": async_request_deepspeed_mii,\n    \"openai\": async_request_openai_completions,\n    \"tensorrt-llm\": async_request_trt_llm,\n}\n", "path": "benchmarks/backend_request_func.py"}], "after_files": [{"content": "import json\nimport os\nimport time\nfrom dataclasses import dataclass\nfrom typing import Optional\n\nimport aiohttp\nfrom tqdm.asyncio import tqdm\n\nAIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)\n\n\n@dataclass\nclass RequestFuncInput:\n    prompt: str\n    api_url: str\n    prompt_len: int\n    output_len: int\n    model: str\n    best_of: int = 1\n    use_beam_search: bool = False\n\n\n@dataclass\nclass RequestFuncOutput:\n    generated_text: str = \"\"\n    success: bool = False\n    latency: float = 0\n    ttft: float = 0\n    prompt_len: int = 0\n\n\nasync def async_request_tgi(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"generate_stream\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert not request_func_input.use_beam_search\n        params = {\n            \"best_of\": request_func_input.best_of,\n            \"max_new_tokens\": request_func_input.output_len,\n            \"do_sample\": True,\n            \"temperature\": 0.01,  # TGI does not accept 0.0 temperature.\n            \"top_p\": 0.99,  # TGI does not accept 1.0 top_p.\n        }\n        payload = {\n            \"inputs\": request_func_input.prompt,\n            \"parameters\": params,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        ttft = 0\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload) as response:\n                if response.status == 200:\n                    async for data in response.content.iter_any():\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n                    output.latency = time.perf_counter() - st\n\n                    body = data.decode(\"utf-8\").lstrip(\"data:\")\n                    output.generated_text = json.loads(body)[\"generated_text\"]\n                    output.success = True\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_vllm(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"generate\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        payload = {\n            \"prompt\": request_func_input.prompt,\n            \"n\": 1,\n            \"best_of\": request_func_input.best_of,\n            \"use_beam_search\": request_func_input.use_beam_search,\n            \"temperature\": 0.0 if request_func_input.use_beam_search else 1.0,\n            \"top_p\": 1.0,\n            \"max_tokens\": request_func_input.output_len,\n            \"ignore_eos\": True,\n            \"stream\": True,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        ttft = 0\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload) as response:\n                if response.status == 200:\n                    async for data in response.content.iter_any():\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n                    output.latency = time.perf_counter() - st\n\n                    # When streaming, '\\0' is appended to the end of the response.\n                    body = data.decode(\"utf-8\").strip(\"\\0\")\n                    output.generated_text = json.loads(\n                        body)[\"text\"][0][len(request_func_input.prompt):]\n                    output.success = True\n\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_trt_llm(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"generate_stream\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert not request_func_input.use_beam_search\n        assert request_func_input.best_of == 1\n        payload = {\n            \"accumulate_tokens\": True,\n            \"text_input\": request_func_input.prompt,\n            \"temperature\": 0.0,\n            \"top_p\": 1.0,\n            \"max_tokens\": request_func_input.output_len,\n            \"stream\": True,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n        ttft = 0\n\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload) as resp:\n                if resp.status == 200:\n                    async for data in resp.content.iter_any():\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n                    output.latency = time.perf_counter() - st\n\n                    body = data.decode(\"utf-8\").lstrip(\"data:\")\n                    output.generated_text = json.loads(body)[\"text_output\"]\n                    output.success = True\n\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_deepspeed_mii(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert request_func_input.best_of == 1\n        assert not request_func_input.use_beam_search\n\n        payload = {\n            \"prompts\": request_func_input.prompt,\n            \"max_new_tokens\": request_func_input.output_len,\n            \"ignore_eos\": True,\n            \"do_sample\": True,\n            \"temperature\":\n            0.01,  # deepspeed-mii does not accept 0.0 temperature.\n            \"top_p\": 1.0,\n        }\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        # DeepSpeed-MII doesn't support streaming as of Jan 28 2024, will use 0 as placeholder.\n        # https://github.com/microsoft/DeepSpeed-MII/pull/311\n        output.ttft = 0\n\n        st = time.perf_counter()\n        try:\n            async with session.post(url=request_func_input.api_url,\n                                    json=payload) as resp:\n                if resp.status == 200:\n                    parsed_resp = await resp.json()\n                    output.latency = time.perf_counter() - st\n                    output.generated_text = parsed_resp[0][\"generated_text\"]\n                    output.success = True\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n        if pbar:\n            pbar.update(1)\n        return output\n\n\nasync def async_request_openai_completions(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\"v1/completions\")\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert not request_func_input.use_beam_search\n        payload = {\n            \"model\": request_func_input.model,\n            \"prompt\": request_func_input.prompt,\n            \"temperature\": 0.0,\n            \"best_of\": request_func_input.best_of,\n            \"max_tokens\": request_func_input.output_len,\n            \"stream\": True,\n        }\n        headers = {\n            \"Authorization\": f\"Bearer {os.environ.get('OPENAI_API_KEY')}\"\n        }\n\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        generated_text = \"\"\n        ttft = 0\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload,\n                                    headers=headers) as response:\n                if response.status == 200:\n                    async for chunk in response.content:\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n\n                        chunk = chunk.strip()\n                        if not chunk:\n                            continue\n\n                        chunk = chunk.decode(\"utf-8\").lstrip(\"data: \")\n                        if chunk == \"[DONE]\":\n                            latency = time.perf_counter() - st\n                        else:\n                            body = json.loads(chunk)\n                            generated_text += body[\"choices\"][0][\"text\"]\n\n                    output.generated_text = generated_text\n                    output.success = True\n                    output.latency = latency\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n    if pbar:\n        pbar.update(1)\n    return output\n\n\nasync def async_request_openai_chat_completions(\n    request_func_input: RequestFuncInput,\n    pbar: Optional[tqdm] = None,\n) -> RequestFuncOutput:\n    api_url = request_func_input.api_url\n    assert api_url.endswith(\n        \"v1/chat/completions\"\n    ), \"OpenAI Chat API URL must end with 'v1/chat/completions'.\"\n\n    async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:\n        assert not request_func_input.use_beam_search\n        payload = {\n            \"model\": request_func_input.model,\n            \"messages\": [\n                {\n                    \"role\": \"user\",\n                    \"content\": request_func_input.prompt,\n                },\n            ],\n            \"temperature\": 0.0,\n            \"max_tokens\": request_func_input.output_len,\n            \"stream\": True,\n        }\n        headers = {\n            \"Content-Type\": \"application/json\",\n            \"Authorization\": f\"Bearer {os.environ.get('OPENAI_API_KEY')}\"\n        }\n\n        output = RequestFuncOutput()\n        output.prompt_len = request_func_input.prompt_len\n\n        generated_text = \"\"\n        ttft = 0\n        st = time.perf_counter()\n        try:\n            async with session.post(url=api_url, json=payload,\n                                    headers=headers) as response:\n                if response.status == 200:\n                    async for chunk in response.content:\n                        if ttft == 0:\n                            ttft = time.perf_counter() - st\n                            output.ttft = ttft\n\n                        chunk = chunk.strip()\n                        if not chunk:\n                            continue\n\n                        chunk = chunk.decode(\"utf-8\").lstrip(\"data: \")\n                        if chunk == \"[DONE]\":\n                            latency = time.perf_counter() - st\n                        else:\n                            body = json.loads(chunk)\n                            if \"content\" in body[\"choices\"][0][\"delta\"]:\n                                generated_text += body[\"choices\"][0][\"delta\"][\n                                    \"content\"]\n\n                    output.generated_text = generated_text\n                    output.success = True\n                    output.latency = latency\n                else:\n                    output.success = False\n        except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):\n            output.success = False\n\n    if pbar:\n        pbar.update(1)\n    return output\n\n\nASYNC_REQUEST_FUNCS = {\n    \"tgi\": async_request_tgi,\n    \"vllm\": async_request_vllm,\n    \"deepspeed-mii\": async_request_deepspeed_mii,\n    \"openai\": async_request_openai_completions,\n    \"openai-chat\": async_request_openai_chat_completions,\n    \"tensorrt-llm\": async_request_trt_llm,\n}\n", "path": "benchmarks/backend_request_func.py"}]}