We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Got OOM when using TP
I got this when running OPT-13B on two A100s, with FP16. The error didn't occur when I decreased `gpu_memory_utilization` to 0.9. It seems our memory profiling is somehow inaccurate when using TP.

Find the command and error msg below:
```
$ python benchmarks/benchmark_latency.py --model facebook/opt-13b -tp 2
Namespace(model='facebook/opt-13b', tensor_parallel_size=2, input_len=32, output_len=128, batch_size=8, n=1, use_beam_search=False, num_iters=3, profile=False)
2023-06-09 09:17:47,945 INFO worker.py:1625 -- Started a local Ray instance.
INFO 06-09 09:17:48 llm_server.py:60] Initializing an LLM server with config: model='facebook/opt-13b', dtype=torch.float16, use_dummy_weights=False, download_dir=None, use_np_weights=False, tensor_parallel_size=2, seed=0)
INFO 06-09 09:18:11 llm_server.py:129] # GPU blocks: 4150, # CPU blocks: 655
Traceback (most recent call last):
  File "/home/gcpuser/workspace/cacheflow/benchmarks/benchmark_latency.py", line 80, in <module>
    main(args)
  File "/home/gcpuser/workspace/cacheflow/benchmarks/benchmark_latency.py", line 17, in main
    llm = LLM(
  File "/home/gcpuser/workspace/cacheflow/cacheflow/entrypoints/llm.py", line 55, in __init__
    self.llm_server = LLMServer.from_server_args(server_args)
  File "/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py", line 146, in from_server_args
    server = cls(*server_configs, distributed_init_method, devices,
  File "/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py", line 103, in __init__
    self._init_cache()
  File "/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py", line 135, in _init_cache
    self._run_workers("init_cache_engine", cache_config=self.cache_config)
  File "/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py", line 312, in _run_workers
    all_outputs = ray.get(all_outputs)
  File "/opt/conda/envs/dev/lib/python3.9/site-packages/ray/_private/client_mode_hook.py", line 105, in wrapper
    return func(*args, **kwargs)
  File "/opt/conda/envs/dev/lib/python3.9/site-packages/ray/_private/worker.py", line 2521, in get
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(OutOfMemoryError): ray::Worker.init_cache_engine() (pid=11306, ip=10.128.0.79, repr=<cacheflow.worker.worker.Worker object at 0x7f4aa97596d0>)
  File "/home/gcpuser/workspace/cacheflow/cacheflow/worker/worker.py", line 127, in init_cache_engine
    self.cache_engine = CacheEngine(
  File "/home/gcpuser/workspace/cacheflow/cacheflow/worker/cache_engine.py", line 40, in __init__
    self.gpu_cache = self.allocate_gpu_cache()
  File "/home/gcpuser/workspace/cacheflow/cacheflow/worker/cache_engine.py", line 76, in allocate_gpu_cache
    value_blocks = torch.empty(
torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 326.00 MiB (GPU 0; 39.41 GiB total capacity; 37.08 GiB already allocated; 188.56 MiB free; 37.25 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
```



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/engine/arg_utils.py`
Content:
```
1 import argparse
2 import dataclasses
3 from dataclasses import dataclass
4 from typing import Optional, Tuple
5 
6 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7                          SchedulerConfig)
8 
9 
10 @dataclass
11 class EngineArgs:
12     """Arguments for vLLM engine."""
13     model: str
14     download_dir: Optional[str] = None
15     use_np_weights: bool = False
16     use_dummy_weights: bool = False
17     dtype: str = "auto"
18     seed: int = 0
19     worker_use_ray: bool = False
20     pipeline_parallel_size: int = 1
21     tensor_parallel_size: int = 1
22     block_size: int = 16
23     swap_space: int = 4  # GiB
24     gpu_memory_utilization: float = 0.95
25     max_num_batched_tokens: int = 2560
26     max_num_seqs: int = 256
27     disable_log_stats: bool = False
28 
29     def __post_init__(self):
30         self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)
31 
32     @staticmethod
33     def add_cli_args(
34         parser: argparse.ArgumentParser,
35     ) -> argparse.ArgumentParser:
36         """Shared CLI arguments for vLLM engine."""
37         # Model arguments
38         parser.add_argument('--model', type=str, default='facebook/opt-125m',
39                             help='name or path of the huggingface model to use')
40         parser.add_argument('--download-dir', type=str,
41                             default=EngineArgs.download_dir,
42                             help='directory to download and load the weights, '
43                                  'default to the default cache dir of '
44                                  'huggingface')
45         parser.add_argument('--use-np-weights', action='store_true',
46                             help='save a numpy copy of model weights for '
47                                  'faster loading. This can increase the disk '
48                                  'usage by up to 2x.')
49         parser.add_argument('--use-dummy-weights', action='store_true',
50                             help='use dummy values for model weights')
51         # TODO(woosuk): Support FP32.
52         parser.add_argument('--dtype', type=str, default=EngineArgs.dtype,
53                             choices=['auto', 'half', 'bfloat16', 'float'],
54                             help='data type for model weights and activations. '
55                                  'The "auto" option will use FP16 precision '
56                                  'for FP32 and FP16 models, and BF16 precision '
57                                  'for BF16 models.')
58         # Parallel arguments
59         parser.add_argument('--worker-use-ray', action='store_true',
60                             help='use Ray for distributed serving, will be '
61                                  'automatically set when using more than 1 GPU')
62         parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
63                             default=EngineArgs.pipeline_parallel_size,
64                             help='number of pipeline stages')
65         parser.add_argument('--tensor-parallel-size', '-tp', type=int,
66                             default=EngineArgs.tensor_parallel_size,
67                             help='number of tensor parallel replicas')
68         # KV cache arguments
69         parser.add_argument('--block-size', type=int,
70                             default=EngineArgs.block_size,
71                             choices=[8, 16, 32],
72                             help='token block size')
73         # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
74         parser.add_argument('--seed', type=int, default=EngineArgs.seed,
75                             help='random seed')
76         parser.add_argument('--swap-space', type=int,
77                             default=EngineArgs.swap_space,
78                             help='CPU swap space size (GiB) per GPU')
79         parser.add_argument('--gpu-memory-utilization', type=float,
80                             default=EngineArgs.gpu_memory_utilization,
81                             help='the percentage of GPU memory to be used for'
82                                  'the model executor')
83         parser.add_argument('--max-num-batched-tokens', type=int,
84                             default=EngineArgs.max_num_batched_tokens,
85                             help='maximum number of batched tokens per '
86                                  'iteration')
87         parser.add_argument('--max-num-seqs', type=int,
88                             default=EngineArgs.max_num_seqs,
89                             help='maximum number of sequences per iteration')
90         parser.add_argument('--disable-log-stats', action='store_true',
91                             help='disable logging statistics')
92         return parser
93 
94     @classmethod
95     def from_cli_args(cls, args: argparse.Namespace) -> "EngineArgs":
96         # Get the list of attributes of this dataclass.
97         attrs = [attr.name for attr in dataclasses.fields(cls)]
98         # Set the attributes from the parsed arguments.
99         engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
100         return engine_args
101 
102     def create_engine_configs(
103         self,
104     ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
105         # Initialize the configs.
106         model_config = ModelConfig(
107             self.model, self.download_dir, self.use_np_weights,
108             self.use_dummy_weights, self.dtype, self.seed)
109         cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
110                                    self.swap_space)
111         parallel_config = ParallelConfig(self.pipeline_parallel_size,
112                                          self.tensor_parallel_size,
113                                          self.worker_use_ray)
114         scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
115                                            self.max_num_seqs)
116         return model_config, cache_config, parallel_config, scheduler_config
117 
118 
119 @dataclass
120 class AsyncEngineArgs(EngineArgs):
121     """Arguments for asynchronous vLLM engine."""
122     engine_use_ray: bool = False
123     disable_log_requests: bool = False
124 
125     @staticmethod
126     def add_cli_args(
127         parser: argparse.ArgumentParser,
128     ) -> argparse.ArgumentParser:
129         parser = EngineArgs.add_cli_args(parser)
130         parser.add_argument('--engine-use-ray', action='store_true',
131                             help='use Ray to start the LLM engine in a '
132                                  'separate process as the server process.')
133         parser.add_argument('--disable-log-requests', action='store_true',
134                             help='disable logging requests')
135         return parser
136
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -21,7 +21,7 @@
     tensor_parallel_size: int = 1
     block_size: int = 16
     swap_space: int = 4  # GiB
-    gpu_memory_utilization: float = 0.95
+    gpu_memory_utilization: float = 0.90
     max_num_batched_tokens: int = 2560
     max_num_seqs: int = 256
     disable_log_stats: bool = False

{"golden_diff": "diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py\n--- a/vllm/engine/arg_utils.py\n+++ b/vllm/engine/arg_utils.py\n@@ -21,7 +21,7 @@\n     tensor_parallel_size: int = 1\n     block_size: int = 16\n     swap_space: int = 4  # GiB\n-    gpu_memory_utilization: float = 0.95\n+    gpu_memory_utilization: float = 0.90\n     max_num_batched_tokens: int = 2560\n     max_num_seqs: int = 256\n     disable_log_stats: bool = False\n", "issue": "Got OOM when using TP\nI got this when running OPT-13B on two A100s, with FP16. The error didn't occur when I decreased `gpu_memory_utilization` to 0.9. It seems our memory profiling is somehow inaccurate when using TP.\r\n\r\nFind the command and error msg below:\r\n```\r\n$ python benchmarks/benchmark_latency.py --model facebook/opt-13b -tp 2\r\nNamespace(model='facebook/opt-13b', tensor_parallel_size=2, input_len=32, output_len=128, batch_size=8, n=1, use_beam_search=False, num_iters=3, profile=False)\r\n2023-06-09 09:17:47,945 INFO worker.py:1625 -- Started a local Ray instance.\r\nINFO 06-09 09:17:48 llm_server.py:60] Initializing an LLM server with config: model='facebook/opt-13b', dtype=torch.float16, use_dummy_weights=False, download_dir=None, use_np_weights=False, tensor_parallel_size=2, seed=0)\r\nINFO 06-09 09:18:11 llm_server.py:129] # GPU blocks: 4150, # CPU blocks: 655\r\nTraceback (most recent call last):\r\n  File \"/home/gcpuser/workspace/cacheflow/benchmarks/benchmark_latency.py\", line 80, in <module>\r\n    main(args)\r\n  File \"/home/gcpuser/workspace/cacheflow/benchmarks/benchmark_latency.py\", line 17, in main\r\n    llm = LLM(\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/entrypoints/llm.py\", line 55, in __init__\r\n    self.llm_server = LLMServer.from_server_args(server_args)\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py\", line 146, in from_server_args\r\n    server = cls(*server_configs, distributed_init_method, devices,\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py\", line 103, in __init__\r\n    self._init_cache()\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py\", line 135, in _init_cache\r\n    self._run_workers(\"init_cache_engine\", cache_config=self.cache_config)\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/server/llm_server.py\", line 312, in _run_workers\r\n    all_outputs = ray.get(all_outputs)\r\n  File \"/opt/conda/envs/dev/lib/python3.9/site-packages/ray/_private/client_mode_hook.py\", line 105, in wrapper\r\n    return func(*args, **kwargs)\r\n  File \"/opt/conda/envs/dev/lib/python3.9/site-packages/ray/_private/worker.py\", line 2521, in get\r\n    raise value.as_instanceof_cause()\r\nray.exceptions.RayTaskError(OutOfMemoryError): ray::Worker.init_cache_engine() (pid=11306, ip=10.128.0.79, repr=<cacheflow.worker.worker.Worker object at 0x7f4aa97596d0>)\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/worker/worker.py\", line 127, in init_cache_engine\r\n    self.cache_engine = CacheEngine(\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/worker/cache_engine.py\", line 40, in __init__\r\n    self.gpu_cache = self.allocate_gpu_cache()\r\n  File \"/home/gcpuser/workspace/cacheflow/cacheflow/worker/cache_engine.py\", line 76, in allocate_gpu_cache\r\n    value_blocks = torch.empty(\r\ntorch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 326.00 MiB (GPU 0; 39.41 GiB total capacity; 37.08 GiB already allocated; 188.56 MiB free; 37.25 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\r\n```\r\n\r\n\n", "before_files": [{"content": "import argparse\nimport dataclasses\nfrom dataclasses import dataclass\nfrom typing import Optional, Tuple\n\nfrom vllm.config import (CacheConfig, ModelConfig, ParallelConfig,\n                         SchedulerConfig)\n\n\n@dataclass\nclass EngineArgs:\n    \"\"\"Arguments for vLLM engine.\"\"\"\n    model: str\n    download_dir: Optional[str] = None\n    use_np_weights: bool = False\n    use_dummy_weights: bool = False\n    dtype: str = \"auto\"\n    seed: int = 0\n    worker_use_ray: bool = False\n    pipeline_parallel_size: int = 1\n    tensor_parallel_size: int = 1\n    block_size: int = 16\n    swap_space: int = 4  # GiB\n    gpu_memory_utilization: float = 0.95\n    max_num_batched_tokens: int = 2560\n    max_num_seqs: int = 256\n    disable_log_stats: bool = False\n\n    def __post_init__(self):\n        self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)\n\n    @staticmethod\n    def add_cli_args(\n        parser: argparse.ArgumentParser,\n    ) -> argparse.ArgumentParser:\n        \"\"\"Shared CLI arguments for vLLM engine.\"\"\"\n        # Model arguments\n        parser.add_argument('--model', type=str, default='facebook/opt-125m',\n                            help='name or path of the huggingface model to use')\n        parser.add_argument('--download-dir', type=str,\n                            default=EngineArgs.download_dir,\n                            help='directory to download and load the weights, '\n                                 'default to the default cache dir of '\n                                 'huggingface')\n        parser.add_argument('--use-np-weights', action='store_true',\n                            help='save a numpy copy of model weights for '\n                                 'faster loading. This can increase the disk '\n                                 'usage by up to 2x.')\n        parser.add_argument('--use-dummy-weights', action='store_true',\n                            help='use dummy values for model weights')\n        # TODO(woosuk): Support FP32.\n        parser.add_argument('--dtype', type=str, default=EngineArgs.dtype,\n                            choices=['auto', 'half', 'bfloat16', 'float'],\n                            help='data type for model weights and activations. '\n                                 'The \"auto\" option will use FP16 precision '\n                                 'for FP32 and FP16 models, and BF16 precision '\n                                 'for BF16 models.')\n        # Parallel arguments\n        parser.add_argument('--worker-use-ray', action='store_true',\n                            help='use Ray for distributed serving, will be '\n                                 'automatically set when using more than 1 GPU')\n        parser.add_argument('--pipeline-parallel-size', '-pp', type=int,\n                            default=EngineArgs.pipeline_parallel_size,\n                            help='number of pipeline stages')\n        parser.add_argument('--tensor-parallel-size', '-tp', type=int,\n                            default=EngineArgs.tensor_parallel_size,\n                            help='number of tensor parallel replicas')\n        # KV cache arguments\n        parser.add_argument('--block-size', type=int,\n                            default=EngineArgs.block_size,\n                            choices=[8, 16, 32],\n                            help='token block size')\n        # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).\n        parser.add_argument('--seed', type=int, default=EngineArgs.seed,\n                            help='random seed')\n        parser.add_argument('--swap-space', type=int,\n                            default=EngineArgs.swap_space,\n                            help='CPU swap space size (GiB) per GPU')\n        parser.add_argument('--gpu-memory-utilization', type=float,\n                            default=EngineArgs.gpu_memory_utilization,\n                            help='the percentage of GPU memory to be used for'\n                                 'the model executor')\n        parser.add_argument('--max-num-batched-tokens', type=int,\n                            default=EngineArgs.max_num_batched_tokens,\n                            help='maximum number of batched tokens per '\n                                 'iteration')\n        parser.add_argument('--max-num-seqs', type=int,\n                            default=EngineArgs.max_num_seqs,\n                            help='maximum number of sequences per iteration')\n        parser.add_argument('--disable-log-stats', action='store_true',\n                            help='disable logging statistics')\n        return parser\n\n    @classmethod\n    def from_cli_args(cls, args: argparse.Namespace) -> \"EngineArgs\":\n        # Get the list of attributes of this dataclass.\n        attrs = [attr.name for attr in dataclasses.fields(cls)]\n        # Set the attributes from the parsed arguments.\n        engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})\n        return engine_args\n\n    def create_engine_configs(\n        self,\n    ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:\n        # Initialize the configs.\n        model_config = ModelConfig(\n            self.model, self.download_dir, self.use_np_weights,\n            self.use_dummy_weights, self.dtype, self.seed)\n        cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,\n                                   self.swap_space)\n        parallel_config = ParallelConfig(self.pipeline_parallel_size,\n                                         self.tensor_parallel_size,\n                                         self.worker_use_ray)\n        scheduler_config = SchedulerConfig(self.max_num_batched_tokens,\n                                           self.max_num_seqs)\n        return model_config, cache_config, parallel_config, scheduler_config\n\n\n@dataclass\nclass AsyncEngineArgs(EngineArgs):\n    \"\"\"Arguments for asynchronous vLLM engine.\"\"\"\n    engine_use_ray: bool = False\n    disable_log_requests: bool = False\n\n    @staticmethod\n    def add_cli_args(\n        parser: argparse.ArgumentParser,\n    ) -> argparse.ArgumentParser:\n        parser = EngineArgs.add_cli_args(parser)\n        parser.add_argument('--engine-use-ray', action='store_true',\n                            help='use Ray to start the LLM engine in a '\n                                 'separate process as the server process.')\n        parser.add_argument('--disable-log-requests', action='store_true',\n                            help='disable logging requests')\n        return parser\n", "path": "vllm/engine/arg_utils.py"}], "after_files": [{"content": "import argparse\nimport dataclasses\nfrom dataclasses import dataclass\nfrom typing import Optional, Tuple\n\nfrom vllm.config import (CacheConfig, ModelConfig, ParallelConfig,\n                         SchedulerConfig)\n\n\n@dataclass\nclass EngineArgs:\n    \"\"\"Arguments for vLLM engine.\"\"\"\n    model: str\n    download_dir: Optional[str] = None\n    use_np_weights: bool = False\n    use_dummy_weights: bool = False\n    dtype: str = \"auto\"\n    seed: int = 0\n    worker_use_ray: bool = False\n    pipeline_parallel_size: int = 1\n    tensor_parallel_size: int = 1\n    block_size: int = 16\n    swap_space: int = 4  # GiB\n    gpu_memory_utilization: float = 0.90\n    max_num_batched_tokens: int = 2560\n    max_num_seqs: int = 256\n    disable_log_stats: bool = False\n\n    def __post_init__(self):\n        self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)\n\n    @staticmethod\n    def add_cli_args(\n        parser: argparse.ArgumentParser,\n    ) -> argparse.ArgumentParser:\n        \"\"\"Shared CLI arguments for vLLM engine.\"\"\"\n        # Model arguments\n        parser.add_argument('--model', type=str, default='facebook/opt-125m',\n                            help='name or path of the huggingface model to use')\n        parser.add_argument('--download-dir', type=str,\n                            default=EngineArgs.download_dir,\n                            help='directory to download and load the weights, '\n                                 'default to the default cache dir of '\n                                 'huggingface')\n        parser.add_argument('--use-np-weights', action='store_true',\n                            help='save a numpy copy of model weights for '\n                                 'faster loading. This can increase the disk '\n                                 'usage by up to 2x.')\n        parser.add_argument('--use-dummy-weights', action='store_true',\n                            help='use dummy values for model weights')\n        # TODO(woosuk): Support FP32.\n        parser.add_argument('--dtype', type=str, default=EngineArgs.dtype,\n                            choices=['auto', 'half', 'bfloat16', 'float'],\n                            help='data type for model weights and activations. '\n                                 'The \"auto\" option will use FP16 precision '\n                                 'for FP32 and FP16 models, and BF16 precision '\n                                 'for BF16 models.')\n        # Parallel arguments\n        parser.add_argument('--worker-use-ray', action='store_true',\n                            help='use Ray for distributed serving, will be '\n                                 'automatically set when using more than 1 GPU')\n        parser.add_argument('--pipeline-parallel-size', '-pp', type=int,\n                            default=EngineArgs.pipeline_parallel_size,\n                            help='number of pipeline stages')\n        parser.add_argument('--tensor-parallel-size', '-tp', type=int,\n                            default=EngineArgs.tensor_parallel_size,\n                            help='number of tensor parallel replicas')\n        # KV cache arguments\n        parser.add_argument('--block-size', type=int,\n                            default=EngineArgs.block_size,\n                            choices=[8, 16, 32],\n                            help='token block size')\n        # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).\n        parser.add_argument('--seed', type=int, default=EngineArgs.seed,\n                            help='random seed')\n        parser.add_argument('--swap-space', type=int,\n                            default=EngineArgs.swap_space,\n                            help='CPU swap space size (GiB) per GPU')\n        parser.add_argument('--gpu-memory-utilization', type=float,\n                            default=EngineArgs.gpu_memory_utilization,\n                            help='the percentage of GPU memory to be used for'\n                                 'the model executor')\n        parser.add_argument('--max-num-batched-tokens', type=int,\n                            default=EngineArgs.max_num_batched_tokens,\n                            help='maximum number of batched tokens per '\n                                 'iteration')\n        parser.add_argument('--max-num-seqs', type=int,\n                            default=EngineArgs.max_num_seqs,\n                            help='maximum number of sequences per iteration')\n        parser.add_argument('--disable-log-stats', action='store_true',\n                            help='disable logging statistics')\n        return parser\n\n    @classmethod\n    def from_cli_args(cls, args: argparse.Namespace) -> \"EngineArgs\":\n        # Get the list of attributes of this dataclass.\n        attrs = [attr.name for attr in dataclasses.fields(cls)]\n        # Set the attributes from the parsed arguments.\n        engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})\n        return engine_args\n\n    def create_engine_configs(\n        self,\n    ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:\n        # Initialize the configs.\n        model_config = ModelConfig(\n            self.model, self.download_dir, self.use_np_weights,\n            self.use_dummy_weights, self.dtype, self.seed)\n        cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,\n                                   self.swap_space)\n        parallel_config = ParallelConfig(self.pipeline_parallel_size,\n                                         self.tensor_parallel_size,\n                                         self.worker_use_ray)\n        scheduler_config = SchedulerConfig(self.max_num_batched_tokens,\n                                           self.max_num_seqs)\n        return model_config, cache_config, parallel_config, scheduler_config\n\n\n@dataclass\nclass AsyncEngineArgs(EngineArgs):\n    \"\"\"Arguments for asynchronous vLLM engine.\"\"\"\n    engine_use_ray: bool = False\n    disable_log_requests: bool = False\n\n    @staticmethod\n    def add_cli_args(\n        parser: argparse.ArgumentParser,\n    ) -> argparse.ArgumentParser:\n        parser = EngineArgs.add_cli_args(parser)\n        parser.add_argument('--engine-use-ray', action='store_true',\n                            help='use Ray to start the LLM engine in a '\n                                 'separate process as the server process.')\n        parser.add_argument('--disable-log-requests', action='store_true',\n                            help='disable logging requests')\n        return parser\n", "path": "vllm/engine/arg_utils.py"}]}