We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Support multiple sampling params in `LLM` API
Hi,

sglang support parallelism [link](https://github.com/sgl-project/sglang#parallelism).

Like the example in the link, can I call the API with different sampling parameters in parallel?

for example

if I have a batched data ; 

```
data=["hi","hello","i'm your assistant"]
```
I want to set temperature as 1.0 for data[0] and 0.7 for data[1] and 0.0 for data[2] then call them simultaneously.




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/entrypoints/llm.py`
Content:
```
1 from typing import List, Optional, Union
2 
3 import torch
4 from tqdm import tqdm
5 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6 
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.llm_engine import LLMEngine
9 from vllm.lora.request import LoRARequest
10 from vllm.outputs import RequestOutput
11 from vllm.sampling_params import SamplingParams
12 from vllm.sequence import MultiModalData
13 from vllm.usage.usage_lib import UsageContext
14 from vllm.utils import Counter
15 
16 
17 class LLM:
18     """An LLM for generating texts from given prompts and sampling parameters.
19 
20     This class includes a tokenizer, a language model (possibly distributed
21     across multiple GPUs), and GPU memory space allocated for intermediate
22     states (aka KV cache). Given a batch of prompts and sampling parameters,
23     this class generates texts from the model, using an intelligent batching
24     mechanism and efficient memory management.
25 
26     NOTE: This class is intended to be used for offline inference. For online
27     serving, use the `AsyncLLMEngine` class instead.
28     NOTE: For the comprehensive list of arguments, see `EngineArgs`.
29 
30     Args:
31         model: The name or path of a HuggingFace Transformers model.
32         tokenizer: The name or path of a HuggingFace Transformers tokenizer.
33         tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
34             if available, and "slow" will always use the slow tokenizer.
35         trust_remote_code: Trust remote code (e.g., from HuggingFace) when
36             downloading the model and tokenizer.
37         tensor_parallel_size: The number of GPUs to use for distributed
38             execution with tensor parallelism.
39         dtype: The data type for the model weights and activations. Currently,
40             we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
41             the `torch_dtype` attribute specified in the model config file.
42             However, if the `torch_dtype` in the config is `float32`, we will
43             use `float16` instead.
44         quantization: The method used to quantize the model weights. Currently,
45             we support "awq", "gptq" and "squeezellm". If None, we first check
46             the `quantization_config` attribute in the model config file. If
47             that is None, we assume the model weights are not quantized and use
48             `dtype` to determine the data type of the weights.
49         revision: The specific model version to use. It can be a branch name,
50             a tag name, or a commit id.
51         tokenizer_revision: The specific tokenizer version to use. It can be a
52             branch name, a tag name, or a commit id.
53         seed: The seed to initialize the random number generator for sampling.
54         gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
55             reserve for the model weights, activations, and KV cache. Higher
56             values will increase the KV cache size and thus improve the model's
57             throughput. However, if the value is too high, it may cause out-of-
58             memory (OOM) errors.
59         swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
60             This can be used for temporarily storing the states of the requests
61             when their `best_of` sampling parameters are larger than 1. If all
62             requests will have `best_of=1`, you can safely set this to 0.
63             Otherwise, too small values may cause out-of-memory (OOM) errors.
64         enforce_eager: Whether to enforce eager execution. If True, we will
65             disable CUDA graph and always execute the model in eager mode.
66             If False, we will use CUDA graph and eager execution in hybrid.
67         max_context_len_to_capture: Maximum context len covered by CUDA graphs.
68             When a sequence has context length larger than this, we fall back
69             to eager mode.
70         disable_custom_all_reduce: See ParallelConfig
71     """
72 
73     def __init__(
74         self,
75         model: str,
76         tokenizer: Optional[str] = None,
77         tokenizer_mode: str = "auto",
78         trust_remote_code: bool = False,
79         tensor_parallel_size: int = 1,
80         dtype: str = "auto",
81         quantization: Optional[str] = None,
82         revision: Optional[str] = None,
83         tokenizer_revision: Optional[str] = None,
84         seed: int = 0,
85         gpu_memory_utilization: float = 0.9,
86         swap_space: int = 4,
87         enforce_eager: bool = False,
88         max_context_len_to_capture: int = 8192,
89         disable_custom_all_reduce: bool = False,
90         **kwargs,
91     ) -> None:
92         if "disable_log_stats" not in kwargs:
93             kwargs["disable_log_stats"] = True
94         engine_args = EngineArgs(
95             model=model,
96             tokenizer=tokenizer,
97             tokenizer_mode=tokenizer_mode,
98             trust_remote_code=trust_remote_code,
99             tensor_parallel_size=tensor_parallel_size,
100             dtype=dtype,
101             quantization=quantization,
102             revision=revision,
103             tokenizer_revision=tokenizer_revision,
104             seed=seed,
105             gpu_memory_utilization=gpu_memory_utilization,
106             swap_space=swap_space,
107             enforce_eager=enforce_eager,
108             max_context_len_to_capture=max_context_len_to_capture,
109             disable_custom_all_reduce=disable_custom_all_reduce,
110             **kwargs,
111         )
112         self.llm_engine = LLMEngine.from_engine_args(
113             engine_args, usage_context=UsageContext.LLM_CLASS)
114         self.request_counter = Counter()
115 
116     def get_tokenizer(
117             self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
118         return self.llm_engine.tokenizer.tokenizer
119 
120     def set_tokenizer(
121         self,
122         tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
123     ) -> None:
124         self.llm_engine.tokenizer.tokenizer = tokenizer
125 
126     def generate(
127         self,
128         prompts: Optional[Union[str, List[str]]] = None,
129         sampling_params: Optional[SamplingParams] = None,
130         prompt_token_ids: Optional[List[List[int]]] = None,
131         use_tqdm: bool = True,
132         lora_request: Optional[LoRARequest] = None,
133         multi_modal_data: Optional[MultiModalData] = None,
134     ) -> List[RequestOutput]:
135         """Generates the completions for the input prompts.
136 
137         NOTE: This class automatically batches the given prompts, considering
138         the memory constraint. For the best performance, put all of your prompts
139         into a single list and pass it to this method.
140 
141         Args:
142             prompts: A list of prompts to generate completions for.
143             sampling_params: The sampling parameters for text generation. If
144                 None, we use the default sampling parameters.
145             prompt_token_ids: A list of token IDs for the prompts. If None, we
146                 use the tokenizer to convert the prompts to token IDs.
147             use_tqdm: Whether to use tqdm to display the progress bar.
148             lora_request: LoRA request to use for generation, if any.
149             multi_modal_data: Multi modal data.
150 
151         Returns:
152             A list of `RequestOutput` objects containing the generated
153             completions in the same order as the input prompts.
154         """
155         if prompts is None and prompt_token_ids is None:
156             raise ValueError("Either prompts or prompt_token_ids must be "
157                              "provided.")
158         if isinstance(prompts, str):
159             # Convert a single prompt to a list.
160             prompts = [prompts]
161         if (prompts is not None and prompt_token_ids is not None
162                 and len(prompts) != len(prompt_token_ids)):
163             raise ValueError("The lengths of prompts and prompt_token_ids "
164                              "must be the same.")
165         if sampling_params is None:
166             # Use default sampling params.
167             sampling_params = SamplingParams()
168 
169         if multi_modal_data:
170             multi_modal_data.data = multi_modal_data.data.to(torch.float16)
171 
172         # Add requests to the engine.
173         if prompts is not None:
174             num_requests = len(prompts)
175         else:
176             assert prompt_token_ids is not None
177             num_requests = len(prompt_token_ids)
178 
179         for i in range(num_requests):
180             prompt = prompts[i] if prompts is not None else None
181             token_ids = None if prompt_token_ids is None else prompt_token_ids[
182                 i]
183             self._add_request(
184                 prompt,
185                 sampling_params,
186                 token_ids,
187                 lora_request=lora_request,
188                 # Get ith image while maintaining the batch dim.
189                 multi_modal_data=MultiModalData(
190                     type=multi_modal_data.type,
191                     data=multi_modal_data.data[i].unsqueeze(0))
192                 if multi_modal_data else None,
193             )
194         return self._run_engine(use_tqdm)
195 
196     def _add_request(
197         self,
198         prompt: Optional[str],
199         sampling_params: SamplingParams,
200         prompt_token_ids: Optional[List[int]],
201         lora_request: Optional[LoRARequest] = None,
202         multi_modal_data: Optional[MultiModalData] = None,
203     ) -> None:
204         request_id = str(next(self.request_counter))
205         self.llm_engine.add_request(request_id,
206                                     prompt,
207                                     sampling_params,
208                                     prompt_token_ids,
209                                     lora_request=lora_request,
210                                     multi_modal_data=multi_modal_data)
211 
212     def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
213         # Initialize tqdm.
214         if use_tqdm:
215             num_requests = self.llm_engine.get_num_unfinished_requests()
216             pbar = tqdm(total=num_requests,
217                         desc="Processed prompts",
218                         dynamic_ncols=True)
219         # Run the engine.
220         outputs: List[RequestOutput] = []
221         while self.llm_engine.has_unfinished_requests():
222             step_outputs = self.llm_engine.step()
223             for output in step_outputs:
224                 if output.finished:
225                     outputs.append(output)
226                     if use_tqdm:
227                         pbar.update(1)
228         if use_tqdm:
229             pbar.close()
230         # Sort the outputs by request ID.
231         # This is necessary because some requests may be finished earlier than
232         # its previous requests.
233         outputs = sorted(outputs, key=lambda x: int(x.request_id))
234         return outputs
235
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py
--- a/vllm/entrypoints/llm.py
+++ b/vllm/entrypoints/llm.py
@@ -126,7 +126,8 @@
     def generate(
         self,
         prompts: Optional[Union[str, List[str]]] = None,
-        sampling_params: Optional[SamplingParams] = None,
+        sampling_params: Optional[Union[SamplingParams,
+                                        List[SamplingParams]]] = None,
         prompt_token_ids: Optional[List[List[int]]] = None,
         use_tqdm: bool = True,
         lora_request: Optional[LoRARequest] = None,
@@ -141,7 +142,10 @@
         Args:
             prompts: A list of prompts to generate completions for.
             sampling_params: The sampling parameters for text generation. If
-                None, we use the default sampling parameters.
+                None, we use the default sampling parameters. 
+                When it is a single value, it is applied to every prompt. 
+                When it is a list, the list must have the same length as the 
+                prompts and it is paired one by one with the prompt.
             prompt_token_ids: A list of token IDs for the prompts. If None, we
                 use the tokenizer to convert the prompts to token IDs.
             use_tqdm: Whether to use tqdm to display the progress bar.
@@ -162,27 +166,33 @@
                 and len(prompts) != len(prompt_token_ids)):
             raise ValueError("The lengths of prompts and prompt_token_ids "
                              "must be the same.")
+
+        if prompts is not None:
+            num_requests = len(prompts)
+        else:
+            assert prompt_token_ids is not None
+            num_requests = len(prompt_token_ids)
+
         if sampling_params is None:
             # Use default sampling params.
             sampling_params = SamplingParams()
 
+        elif isinstance(sampling_params,
+                        list) and len(sampling_params) != num_requests:
+            raise ValueError("The lengths of prompts and sampling_params "
+                             "must be the same.")
         if multi_modal_data:
             multi_modal_data.data = multi_modal_data.data.to(torch.float16)
 
         # Add requests to the engine.
-        if prompts is not None:
-            num_requests = len(prompts)
-        else:
-            assert prompt_token_ids is not None
-            num_requests = len(prompt_token_ids)
-
         for i in range(num_requests):
             prompt = prompts[i] if prompts is not None else None
             token_ids = None if prompt_token_ids is None else prompt_token_ids[
                 i]
             self._add_request(
                 prompt,
-                sampling_params,
+                sampling_params[i]
+                if isinstance(sampling_params, list) else sampling_params,
                 token_ids,
                 lora_request=lora_request,
                 # Get ith image while maintaining the batch dim.
@@ -231,4 +241,4 @@
         # This is necessary because some requests may be finished earlier than
         # its previous requests.
         outputs = sorted(outputs, key=lambda x: int(x.request_id))
-        return outputs
+        return outputs
\ No newline at end of file

{"golden_diff": "diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py\n--- a/vllm/entrypoints/llm.py\n+++ b/vllm/entrypoints/llm.py\n@@ -126,7 +126,8 @@\n     def generate(\n         self,\n         prompts: Optional[Union[str, List[str]]] = None,\n-        sampling_params: Optional[SamplingParams] = None,\n+        sampling_params: Optional[Union[SamplingParams,\n+                                        List[SamplingParams]]] = None,\n         prompt_token_ids: Optional[List[List[int]]] = None,\n         use_tqdm: bool = True,\n         lora_request: Optional[LoRARequest] = None,\n@@ -141,7 +142,10 @@\n         Args:\n             prompts: A list of prompts to generate completions for.\n             sampling_params: The sampling parameters for text generation. If\n-                None, we use the default sampling parameters.\n+                None, we use the default sampling parameters. \n+                When it is a single value, it is applied to every prompt. \n+                When it is a list, the list must have the same length as the \n+                prompts and it is paired one by one with the prompt.\n             prompt_token_ids: A list of token IDs for the prompts. If None, we\n                 use the tokenizer to convert the prompts to token IDs.\n             use_tqdm: Whether to use tqdm to display the progress bar.\n@@ -162,27 +166,33 @@\n                 and len(prompts) != len(prompt_token_ids)):\n             raise ValueError(\"The lengths of prompts and prompt_token_ids \"\n                              \"must be the same.\")\n+\n+        if prompts is not None:\n+            num_requests = len(prompts)\n+        else:\n+            assert prompt_token_ids is not None\n+            num_requests = len(prompt_token_ids)\n+\n         if sampling_params is None:\n             # Use default sampling params.\n             sampling_params = SamplingParams()\n \n+        elif isinstance(sampling_params,\n+                        list) and len(sampling_params) != num_requests:\n+            raise ValueError(\"The lengths of prompts and sampling_params \"\n+                             \"must be the same.\")\n         if multi_modal_data:\n             multi_modal_data.data = multi_modal_data.data.to(torch.float16)\n \n         # Add requests to the engine.\n-        if prompts is not None:\n-            num_requests = len(prompts)\n-        else:\n-            assert prompt_token_ids is not None\n-            num_requests = len(prompt_token_ids)\n-\n         for i in range(num_requests):\n             prompt = prompts[i] if prompts is not None else None\n             token_ids = None if prompt_token_ids is None else prompt_token_ids[\n                 i]\n             self._add_request(\n                 prompt,\n-                sampling_params,\n+                sampling_params[i]\n+                if isinstance(sampling_params, list) else sampling_params,\n                 token_ids,\n                 lora_request=lora_request,\n                 # Get ith image while maintaining the batch dim.\n@@ -231,4 +241,4 @@\n         # This is necessary because some requests may be finished earlier than\n         # its previous requests.\n         outputs = sorted(outputs, key=lambda x: int(x.request_id))\n-        return outputs\n+        return outputs\n\\ No newline at end of file\n", "issue": "Support multiple sampling params in `LLM` API\nHi,\r\n\r\nsglang support parallelism [link](https://github.com/sgl-project/sglang#parallelism).\r\n\r\nLike the example in the link, can I call the API with different sampling parameters in parallel?\r\n\r\nfor example\r\n\r\nif I have a batched data ; \r\n\r\n```\r\ndata=[\"hi\",\"hello\",\"i'm your assistant\"]\r\n```\r\nI want to set temperature as 1.0 for data[0] and 0.7 for data[1] and 0.0 for data[2] then call them simultaneously.\r\n\r\n\r\n\n", "before_files": [{"content": "from typing import List, Optional, Union\n\nimport torch\nfrom tqdm import tqdm\nfrom transformers import PreTrainedTokenizer, PreTrainedTokenizerFast\n\nfrom vllm.engine.arg_utils import EngineArgs\nfrom vllm.engine.llm_engine import LLMEngine\nfrom vllm.lora.request import LoRARequest\nfrom vllm.outputs import RequestOutput\nfrom vllm.sampling_params import SamplingParams\nfrom vllm.sequence import MultiModalData\nfrom vllm.usage.usage_lib import UsageContext\nfrom vllm.utils import Counter\n\n\nclass LLM:\n    \"\"\"An LLM for generating texts from given prompts and sampling parameters.\n\n    This class includes a tokenizer, a language model (possibly distributed\n    across multiple GPUs), and GPU memory space allocated for intermediate\n    states (aka KV cache). Given a batch of prompts and sampling parameters,\n    this class generates texts from the model, using an intelligent batching\n    mechanism and efficient memory management.\n\n    NOTE: This class is intended to be used for offline inference. For online\n    serving, use the `AsyncLLMEngine` class instead.\n    NOTE: For the comprehensive list of arguments, see `EngineArgs`.\n\n    Args:\n        model: The name or path of a HuggingFace Transformers model.\n        tokenizer: The name or path of a HuggingFace Transformers tokenizer.\n        tokenizer_mode: The tokenizer mode. \"auto\" will use the fast tokenizer\n            if available, and \"slow\" will always use the slow tokenizer.\n        trust_remote_code: Trust remote code (e.g., from HuggingFace) when\n            downloading the model and tokenizer.\n        tensor_parallel_size: The number of GPUs to use for distributed\n            execution with tensor parallelism.\n        dtype: The data type for the model weights and activations. Currently,\n            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use\n            the `torch_dtype` attribute specified in the model config file.\n            However, if the `torch_dtype` in the config is `float32`, we will\n            use `float16` instead.\n        quantization: The method used to quantize the model weights. Currently,\n            we support \"awq\", \"gptq\" and \"squeezellm\". If None, we first check\n            the `quantization_config` attribute in the model config file. If\n            that is None, we assume the model weights are not quantized and use\n            `dtype` to determine the data type of the weights.\n        revision: The specific model version to use. It can be a branch name,\n            a tag name, or a commit id.\n        tokenizer_revision: The specific tokenizer version to use. It can be a\n            branch name, a tag name, or a commit id.\n        seed: The seed to initialize the random number generator for sampling.\n        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to\n            reserve for the model weights, activations, and KV cache. Higher\n            values will increase the KV cache size and thus improve the model's\n            throughput. However, if the value is too high, it may cause out-of-\n            memory (OOM) errors.\n        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.\n            This can be used for temporarily storing the states of the requests\n            when their `best_of` sampling parameters are larger than 1. If all\n            requests will have `best_of=1`, you can safely set this to 0.\n            Otherwise, too small values may cause out-of-memory (OOM) errors.\n        enforce_eager: Whether to enforce eager execution. If True, we will\n            disable CUDA graph and always execute the model in eager mode.\n            If False, we will use CUDA graph and eager execution in hybrid.\n        max_context_len_to_capture: Maximum context len covered by CUDA graphs.\n            When a sequence has context length larger than this, we fall back\n            to eager mode.\n        disable_custom_all_reduce: See ParallelConfig\n    \"\"\"\n\n    def __init__(\n        self,\n        model: str,\n        tokenizer: Optional[str] = None,\n        tokenizer_mode: str = \"auto\",\n        trust_remote_code: bool = False,\n        tensor_parallel_size: int = 1,\n        dtype: str = \"auto\",\n        quantization: Optional[str] = None,\n        revision: Optional[str] = None,\n        tokenizer_revision: Optional[str] = None,\n        seed: int = 0,\n        gpu_memory_utilization: float = 0.9,\n        swap_space: int = 4,\n        enforce_eager: bool = False,\n        max_context_len_to_capture: int = 8192,\n        disable_custom_all_reduce: bool = False,\n        **kwargs,\n    ) -> None:\n        if \"disable_log_stats\" not in kwargs:\n            kwargs[\"disable_log_stats\"] = True\n        engine_args = EngineArgs(\n            model=model,\n            tokenizer=tokenizer,\n            tokenizer_mode=tokenizer_mode,\n            trust_remote_code=trust_remote_code,\n            tensor_parallel_size=tensor_parallel_size,\n            dtype=dtype,\n            quantization=quantization,\n            revision=revision,\n            tokenizer_revision=tokenizer_revision,\n            seed=seed,\n            gpu_memory_utilization=gpu_memory_utilization,\n            swap_space=swap_space,\n            enforce_eager=enforce_eager,\n            max_context_len_to_capture=max_context_len_to_capture,\n            disable_custom_all_reduce=disable_custom_all_reduce,\n            **kwargs,\n        )\n        self.llm_engine = LLMEngine.from_engine_args(\n            engine_args, usage_context=UsageContext.LLM_CLASS)\n        self.request_counter = Counter()\n\n    def get_tokenizer(\n            self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:\n        return self.llm_engine.tokenizer.tokenizer\n\n    def set_tokenizer(\n        self,\n        tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],\n    ) -> None:\n        self.llm_engine.tokenizer.tokenizer = tokenizer\n\n    def generate(\n        self,\n        prompts: Optional[Union[str, List[str]]] = None,\n        sampling_params: Optional[SamplingParams] = None,\n        prompt_token_ids: Optional[List[List[int]]] = None,\n        use_tqdm: bool = True,\n        lora_request: Optional[LoRARequest] = None,\n        multi_modal_data: Optional[MultiModalData] = None,\n    ) -> List[RequestOutput]:\n        \"\"\"Generates the completions for the input prompts.\n\n        NOTE: This class automatically batches the given prompts, considering\n        the memory constraint. For the best performance, put all of your prompts\n        into a single list and pass it to this method.\n\n        Args:\n            prompts: A list of prompts to generate completions for.\n            sampling_params: The sampling parameters for text generation. If\n                None, we use the default sampling parameters.\n            prompt_token_ids: A list of token IDs for the prompts. If None, we\n                use the tokenizer to convert the prompts to token IDs.\n            use_tqdm: Whether to use tqdm to display the progress bar.\n            lora_request: LoRA request to use for generation, if any.\n            multi_modal_data: Multi modal data.\n\n        Returns:\n            A list of `RequestOutput` objects containing the generated\n            completions in the same order as the input prompts.\n        \"\"\"\n        if prompts is None and prompt_token_ids is None:\n            raise ValueError(\"Either prompts or prompt_token_ids must be \"\n                             \"provided.\")\n        if isinstance(prompts, str):\n            # Convert a single prompt to a list.\n            prompts = [prompts]\n        if (prompts is not None and prompt_token_ids is not None\n                and len(prompts) != len(prompt_token_ids)):\n            raise ValueError(\"The lengths of prompts and prompt_token_ids \"\n                             \"must be the same.\")\n        if sampling_params is None:\n            # Use default sampling params.\n            sampling_params = SamplingParams()\n\n        if multi_modal_data:\n            multi_modal_data.data = multi_modal_data.data.to(torch.float16)\n\n        # Add requests to the engine.\n        if prompts is not None:\n            num_requests = len(prompts)\n        else:\n            assert prompt_token_ids is not None\n            num_requests = len(prompt_token_ids)\n\n        for i in range(num_requests):\n            prompt = prompts[i] if prompts is not None else None\n            token_ids = None if prompt_token_ids is None else prompt_token_ids[\n                i]\n            self._add_request(\n                prompt,\n                sampling_params,\n                token_ids,\n                lora_request=lora_request,\n                # Get ith image while maintaining the batch dim.\n                multi_modal_data=MultiModalData(\n                    type=multi_modal_data.type,\n                    data=multi_modal_data.data[i].unsqueeze(0))\n                if multi_modal_data else None,\n            )\n        return self._run_engine(use_tqdm)\n\n    def _add_request(\n        self,\n        prompt: Optional[str],\n        sampling_params: SamplingParams,\n        prompt_token_ids: Optional[List[int]],\n        lora_request: Optional[LoRARequest] = None,\n        multi_modal_data: Optional[MultiModalData] = None,\n    ) -> None:\n        request_id = str(next(self.request_counter))\n        self.llm_engine.add_request(request_id,\n                                    prompt,\n                                    sampling_params,\n                                    prompt_token_ids,\n                                    lora_request=lora_request,\n                                    multi_modal_data=multi_modal_data)\n\n    def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:\n        # Initialize tqdm.\n        if use_tqdm:\n            num_requests = self.llm_engine.get_num_unfinished_requests()\n            pbar = tqdm(total=num_requests,\n                        desc=\"Processed prompts\",\n                        dynamic_ncols=True)\n        # Run the engine.\n        outputs: List[RequestOutput] = []\n        while self.llm_engine.has_unfinished_requests():\n            step_outputs = self.llm_engine.step()\n            for output in step_outputs:\n                if output.finished:\n                    outputs.append(output)\n                    if use_tqdm:\n                        pbar.update(1)\n        if use_tqdm:\n            pbar.close()\n        # Sort the outputs by request ID.\n        # This is necessary because some requests may be finished earlier than\n        # its previous requests.\n        outputs = sorted(outputs, key=lambda x: int(x.request_id))\n        return outputs\n", "path": "vllm/entrypoints/llm.py"}], "after_files": [{"content": "from typing import List, Optional, Union\n\nimport torch\nfrom tqdm import tqdm\nfrom transformers import PreTrainedTokenizer, PreTrainedTokenizerFast\n\nfrom vllm.engine.arg_utils import EngineArgs\nfrom vllm.engine.llm_engine import LLMEngine\nfrom vllm.lora.request import LoRARequest\nfrom vllm.outputs import RequestOutput\nfrom vllm.sampling_params import SamplingParams\nfrom vllm.sequence import MultiModalData\nfrom vllm.usage.usage_lib import UsageContext\nfrom vllm.utils import Counter\n\n\nclass LLM:\n    \"\"\"An LLM for generating texts from given prompts and sampling parameters.\n\n    This class includes a tokenizer, a language model (possibly distributed\n    across multiple GPUs), and GPU memory space allocated for intermediate\n    states (aka KV cache). Given a batch of prompts and sampling parameters,\n    this class generates texts from the model, using an intelligent batching\n    mechanism and efficient memory management.\n\n    NOTE: This class is intended to be used for offline inference. For online\n    serving, use the `AsyncLLMEngine` class instead.\n    NOTE: For the comprehensive list of arguments, see `EngineArgs`.\n\n    Args:\n        model: The name or path of a HuggingFace Transformers model.\n        tokenizer: The name or path of a HuggingFace Transformers tokenizer.\n        tokenizer_mode: The tokenizer mode. \"auto\" will use the fast tokenizer\n            if available, and \"slow\" will always use the slow tokenizer.\n        trust_remote_code: Trust remote code (e.g., from HuggingFace) when\n            downloading the model and tokenizer.\n        tensor_parallel_size: The number of GPUs to use for distributed\n            execution with tensor parallelism.\n        dtype: The data type for the model weights and activations. Currently,\n            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use\n            the `torch_dtype` attribute specified in the model config file.\n            However, if the `torch_dtype` in the config is `float32`, we will\n            use `float16` instead.\n        quantization: The method used to quantize the model weights. Currently,\n            we support \"awq\", \"gptq\" and \"squeezellm\". If None, we first check\n            the `quantization_config` attribute in the model config file. If\n            that is None, we assume the model weights are not quantized and use\n            `dtype` to determine the data type of the weights.\n        revision: The specific model version to use. It can be a branch name,\n            a tag name, or a commit id.\n        tokenizer_revision: The specific tokenizer version to use. It can be a\n            branch name, a tag name, or a commit id.\n        seed: The seed to initialize the random number generator for sampling.\n        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to\n            reserve for the model weights, activations, and KV cache. Higher\n            values will increase the KV cache size and thus improve the model's\n            throughput. However, if the value is too high, it may cause out-of-\n            memory (OOM) errors.\n        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.\n            This can be used for temporarily storing the states of the requests\n            when their `best_of` sampling parameters are larger than 1. If all\n            requests will have `best_of=1`, you can safely set this to 0.\n            Otherwise, too small values may cause out-of-memory (OOM) errors.\n        enforce_eager: Whether to enforce eager execution. If True, we will\n            disable CUDA graph and always execute the model in eager mode.\n            If False, we will use CUDA graph and eager execution in hybrid.\n        max_context_len_to_capture: Maximum context len covered by CUDA graphs.\n            When a sequence has context length larger than this, we fall back\n            to eager mode.\n        disable_custom_all_reduce: See ParallelConfig\n    \"\"\"\n\n    def __init__(\n        self,\n        model: str,\n        tokenizer: Optional[str] = None,\n        tokenizer_mode: str = \"auto\",\n        trust_remote_code: bool = False,\n        tensor_parallel_size: int = 1,\n        dtype: str = \"auto\",\n        quantization: Optional[str] = None,\n        revision: Optional[str] = None,\n        tokenizer_revision: Optional[str] = None,\n        seed: int = 0,\n        gpu_memory_utilization: float = 0.9,\n        swap_space: int = 4,\n        enforce_eager: bool = False,\n        max_context_len_to_capture: int = 8192,\n        disable_custom_all_reduce: bool = False,\n        **kwargs,\n    ) -> None:\n        if \"disable_log_stats\" not in kwargs:\n            kwargs[\"disable_log_stats\"] = True\n        engine_args = EngineArgs(\n            model=model,\n            tokenizer=tokenizer,\n            tokenizer_mode=tokenizer_mode,\n            trust_remote_code=trust_remote_code,\n            tensor_parallel_size=tensor_parallel_size,\n            dtype=dtype,\n            quantization=quantization,\n            revision=revision,\n            tokenizer_revision=tokenizer_revision,\n            seed=seed,\n            gpu_memory_utilization=gpu_memory_utilization,\n            swap_space=swap_space,\n            enforce_eager=enforce_eager,\n            max_context_len_to_capture=max_context_len_to_capture,\n            disable_custom_all_reduce=disable_custom_all_reduce,\n            **kwargs,\n        )\n        self.llm_engine = LLMEngine.from_engine_args(\n            engine_args, usage_context=UsageContext.LLM_CLASS)\n        self.request_counter = Counter()\n\n    def get_tokenizer(\n            self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:\n        return self.llm_engine.tokenizer.tokenizer\n\n    def set_tokenizer(\n        self,\n        tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],\n    ) -> None:\n        self.llm_engine.tokenizer.tokenizer = tokenizer\n\n    def generate(\n        self,\n        prompts: Optional[Union[str, List[str]]] = None,\n        sampling_params: Optional[Union[SamplingParams,\n                                        List[SamplingParams]]] = None,\n        prompt_token_ids: Optional[List[List[int]]] = None,\n        use_tqdm: bool = True,\n        lora_request: Optional[LoRARequest] = None,\n        multi_modal_data: Optional[MultiModalData] = None,\n    ) -> List[RequestOutput]:\n        \"\"\"Generates the completions for the input prompts.\n\n        NOTE: This class automatically batches the given prompts, considering\n        the memory constraint. For the best performance, put all of your prompts\n        into a single list and pass it to this method.\n\n        Args:\n            prompts: A list of prompts to generate completions for.\n            sampling_params: The sampling parameters for text generation. If\n                None, we use the default sampling parameters. \n                When it is a single value, it is applied to every prompt. \n                When it is a list, the list must have the same length as the \n                prompts and it is paired one by one with the prompt.\n            prompt_token_ids: A list of token IDs for the prompts. If None, we\n                use the tokenizer to convert the prompts to token IDs.\n            use_tqdm: Whether to use tqdm to display the progress bar.\n            lora_request: LoRA request to use for generation, if any.\n            multi_modal_data: Multi modal data.\n\n        Returns:\n            A list of `RequestOutput` objects containing the generated\n            completions in the same order as the input prompts.\n        \"\"\"\n        if prompts is None and prompt_token_ids is None:\n            raise ValueError(\"Either prompts or prompt_token_ids must be \"\n                             \"provided.\")\n        if isinstance(prompts, str):\n            # Convert a single prompt to a list.\n            prompts = [prompts]\n        if (prompts is not None and prompt_token_ids is not None\n                and len(prompts) != len(prompt_token_ids)):\n            raise ValueError(\"The lengths of prompts and prompt_token_ids \"\n                             \"must be the same.\")\n\n        if prompts is not None:\n            num_requests = len(prompts)\n        else:\n            assert prompt_token_ids is not None\n            num_requests = len(prompt_token_ids)\n\n        if sampling_params is None:\n            # Use default sampling params.\n            sampling_params = SamplingParams()\n\n        elif isinstance(sampling_params,\n                        list) and len(sampling_params) != num_requests:\n            raise ValueError(\"The lengths of prompts and sampling_params \"\n                             \"must be the same.\")\n        if multi_modal_data:\n            multi_modal_data.data = multi_modal_data.data.to(torch.float16)\n\n        # Add requests to the engine.\n        for i in range(num_requests):\n            prompt = prompts[i] if prompts is not None else None\n            token_ids = None if prompt_token_ids is None else prompt_token_ids[\n                i]\n            self._add_request(\n                prompt,\n                sampling_params[i]\n                if isinstance(sampling_params, list) else sampling_params,\n                token_ids,\n                lora_request=lora_request,\n                # Get ith image while maintaining the batch dim.\n                multi_modal_data=MultiModalData(\n                    type=multi_modal_data.type,\n                    data=multi_modal_data.data[i].unsqueeze(0))\n                if multi_modal_data else None,\n            )\n        return self._run_engine(use_tqdm)\n\n    def _add_request(\n        self,\n        prompt: Optional[str],\n        sampling_params: SamplingParams,\n        prompt_token_ids: Optional[List[int]],\n        lora_request: Optional[LoRARequest] = None,\n        multi_modal_data: Optional[MultiModalData] = None,\n    ) -> None:\n        request_id = str(next(self.request_counter))\n        self.llm_engine.add_request(request_id,\n                                    prompt,\n                                    sampling_params,\n                                    prompt_token_ids,\n                                    lora_request=lora_request,\n                                    multi_modal_data=multi_modal_data)\n\n    def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:\n        # Initialize tqdm.\n        if use_tqdm:\n            num_requests = self.llm_engine.get_num_unfinished_requests()\n            pbar = tqdm(total=num_requests,\n                        desc=\"Processed prompts\",\n                        dynamic_ncols=True)\n        # Run the engine.\n        outputs: List[RequestOutput] = []\n        while self.llm_engine.has_unfinished_requests():\n            step_outputs = self.llm_engine.step()\n            for output in step_outputs:\n                if output.finished:\n                    outputs.append(output)\n                    if use_tqdm:\n                        pbar.update(1)\n        if use_tqdm:\n            pbar.close()\n        # Sort the outputs by request ID.\n        # This is necessary because some requests may be finished earlier than\n        # its previous requests.\n        outputs = sorted(outputs, key=lambda x: int(x.request_id))\n        return outputs", "path": "vllm/entrypoints/llm.py"}]}