We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
bug: Triton Integration failed with an AttributeError
### Describe the bug

I try to integrate Triton Inference Server into BentoML and I follow both the official documentation(https://docs.bentoml.org/en/latest/integrations/triton.html) and the following example(https://github.com/bentoml/BentoML/tree/main/examples/triton/onnx). However, I get an AttributeError with the following message:
" '_TritonRunner' object has no attribute 'onnx_fp_16'", where onnx_fp_16 is my model deployed in Triton.
The code in my service.py file is:
```
import bentoml
from bentoml.io import JSON
from src.tokenizer import RDATokenizer
from src.request import RDARequest
from src.response import RDAResponse
from src.config import  load_config
from src.utils.model import load_topics_and_mapping, get_multi_label_binarizer, post_predict, load_taxonomy_root,\
    load_parent_child_relationship

configs = load_config()
rda_triton_runner = bentoml.triton.Runner("rdav2",
                                          model_repository="src/model_repository",
                                          cli_args=["--model-control-mode=explicit",
                                                    "--load-model=onnx_fp_16",
                                                    "--log-verbose=1"])

svc = bentoml.Service("rdav2", runners=[rda_triton_runner])
tokenizer = RDATokenizer(tokenizer_path=configs.tokenizer_path)
labels_binarizer = get_multi_label_binarizer(config=configs)
topics, topics_mapping = load_topics_and_mapping(config=configs)
tax_root = load_taxonomy_root(configs.tax_root_path)
parent_child = load_parent_child_relationship(config=configs)
id_to_fos = {id_: fos for fos, id_ in topics_mapping.items()}


@svc.api(input=JSON(pydantic_model=RDARequest), output=JSON(pydantic_model=RDAResponse))
async def predict(rda_request: RDARequest) -> RDAResponse:
    """
    The input will be the title, abstract, k and, threshold and the output will be the prediction of the model
    :param rda_request: the request object
    :return: a dictionary where the key is the metadata name and the value is the value of the metadata.
    the metadata will be a list with the tags, the tags ids, the probabilities, the ancestros and the ancestors id
    """
    title = rda_request.title
    abstract = rda_request.abstract
    k = rda_request.k
    threshold = rda_request.threshold
    text = title + ". " + abstract
    encoded_text = await tokenizer.tokenize(text=text)
    logits = await rda_triton_runner.onnx_fp_16.async_run(encoded_text)
    logits = logits[0].squeeze()
    response = await post_predict(logits=logits,
                                  labels_binarizer=labels_binarizer,
                                  k=k,
                                  threshold=threshold,
                                  topics=topics,
                                  topics_mapping=topics_mapping,
                                  tax_roots=tax_root,
                                  parent_child=parent_child,
                                  id_to_fos=id_to_fos)
    return response
```
My _bentofile.yaml_ is the following:
```
service: "service:svc"
include:
- "*.py"
- "/model_repository"
- "/configs"
- "/checkpoints"
python:
  requirements_txt: "requirements.txt"
docker:
  base_image: nvcr.io/nvidia/tritonserver:22.12-py3
```
I tried to run the service with a container and the commands I typed to do so were the followings:
- bentoml build --version 0.0.2
- bentoml containerize rdav2:0.0.2
- docker run -it --rm -p 3000:3000 rdav2:0.0.2 serve --production

I guess there is something wrong in __rda_triton_runner__ object, because when I debug the service, I get an empty list in the __models__ parameter. I am not sure about that, and that's the reason I opened this issue.

### To reproduce

_No response_

### Expected behavior

_No response_

### Environment

bentoml[triton]: 1.0.32
python: 3.8
platform: Ubuntu: 20.10

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/bentoml/triton.py`
Content:
```
1 from __future__ import annotations
2 
3 import logging
4 import typing as t
5 from functools import cached_property
6 
7 import attr
8 from simple_di import Provide as _Provide
9 from simple_di import inject as _inject
10 
11 from ._internal.configuration import get_debug_mode as _get_debug_mode
12 from ._internal.configuration.containers import BentoMLContainer as _BentoMLContainer
13 from ._internal.runner.runnable import RunnableMethodConfig as _RunnableMethodConfig
14 from ._internal.runner.runner import AbstractRunner as _AbstractRunner
15 from ._internal.runner.runner import RunnerMethod as _RunnerMethod
16 from ._internal.runner.runner import object_setattr as _object_setattr
17 from ._internal.runner.runner_handle import DummyRunnerHandle as _DummyRunnerHandle
18 from ._internal.runner.runner_handle.remote import TRITON_EXC_MSG as _TRITON_EXC_MSG
19 from ._internal.runner.runner_handle.remote import (
20     handle_triton_exception as _handle_triton_exception,
21 )
22 from ._internal.utils import LazyLoader as _LazyLoader
23 
24 if t.TYPE_CHECKING:
25     import tritonclient.grpc.aio as _tritongrpcclient
26     import tritonclient.http.aio as _tritonhttpclient
27 
28     from ._internal.runner.runner_handle import RunnerHandle
29 
30     _P = t.ParamSpec("_P")
31 
32     _LogFormat = t.Literal["default", "ISO8601"]
33     _GrpcInferResponseCompressionLevel = t.Literal["none", "low", "medium", "high"]
34     _TraceLevel = t.Literal["OFF", "TIMESTAMPS", "TENSORS"]
35     _RateLimit = t.Literal["execution_count", "off"]
36     _TritonServerType = t.Literal["grpc", "http"]
37 
38     _ClientMethod = t.Literal[
39         "get_cuda_shared_memory_status",
40         "get_inference_statistics",
41         "get_log_settings",
42         "get_model_config",
43         "get_model_metadata",
44         "get_model_repository_index",
45         "get_server_metadata",
46         "get_system_shared_memory_status",
47         "get_trace_settings",
48         "infer",
49         "is_model_ready",
50         "is_server_live",
51         "is_server_ready",
52         "load_model",
53         "register_cuda_shared_memory",
54         "register_system_shared_memory",
55         "stream_infer",
56         "unload_model",
57         "unregister_cuda_shared_memory",
58         "unregister_system_shared_memory",
59         "update_log_settings",
60         "update_trace_settings",
61     ]
62     _ModelName = t.Annotated[str, t.LiteralString]
63 
64 else:
65     _P = t.TypeVar("_P")
66 
67     _LogFormat = _GrpcInferResponseCompressionLevel = _TraceLevel = _RateLimit = str
68 
69     _tritongrpcclient = _LazyLoader(
70         "_tritongrpcclient", globals(), "tritonclient.grpc.aio", exc_msg=_TRITON_EXC_MSG
71     )
72     _tritonhttpclient = _LazyLoader(
73         "_tritonhttpclient", globals(), "tritonclient.http.aio", exc_msg=_TRITON_EXC_MSG
74     )
75 
76 _logger = logging.getLogger(__name__)
77 
78 __all__ = ["Runner"]
79 
80 
81 @attr.define(slots=False, frozen=True, eq=False)
82 class _TritonRunner(_AbstractRunner):
83     repository_path: str
84 
85     tritonserver_type: _TritonServerType = attr.field(
86         default="grpc", validator=attr.validators.in_(["grpc", "http"])
87     )
88     cli_args: list[str] = attr.field(factory=list)
89 
90     _runner_handle: RunnerHandle = attr.field(init=False, factory=_DummyRunnerHandle)
91 
92     @_inject
93     async def runner_handle_is_ready(
94         self,
95         timeout: int = _Provide[
96             _BentoMLContainer.api_server_config.runner_probe.timeout
97         ],
98     ) -> bool:
99         """
100         Check if given runner handle is ready. This will be used as readiness probe in Kubernetes.
101         """
102         return await self._runner_handle.is_ready(timeout)
103 
104     def __init__(
105         self,
106         name: str,
107         model_repository: str,
108         tritonserver_type: _TritonServerType = "grpc",
109         cli_args: list[str] | None = None,
110     ):
111         if cli_args is None:
112             cli_args = []
113 
114         cli_args.append(f"--model-repository={model_repository}")
115 
116         if tritonserver_type == "http":
117             cli_args.extend(
118                 [
119                     "--allow-grpc=False",
120                     "--http-address=127.0.0.1",
121                 ]
122             )
123         elif tritonserver_type == "grpc":
124             cli_args.extend(
125                 [
126                     "--reuse-grpc-port=1",
127                     "--allow-http=False",
128                     "--grpc-address=0.0.0.0",
129                 ]
130             )
131 
132         # default settings, disable metrics
133         cli_args.extend([f"--log-verbose={1 if _get_debug_mode() else 0}"])
134 
135         if not all(s.startswith("--") for s in cli_args):
136             raise ValueError(
137                 "cli_args should be a list of strings starting with '--' for TritonRunner."
138             )
139 
140         self.__attrs_init__(
141             name=name,
142             models=None,
143             resource_config=None,
144             runnable_class=self.__class__,
145             repository_path=model_repository,
146             tritonserver_type=tritonserver_type,
147             cli_args=cli_args,
148             embedded=False,  # NOTE: TritonRunner shouldn't be used as embedded.
149         )
150 
151     @cached_property
152     def protocol_address(self):
153         from ._internal.utils import reserve_free_port
154 
155         if self.tritonserver_type == "http":
156             with reserve_free_port(host="127.0.0.1") as port:
157                 pass
158             return f"127.0.0.1:{port}"
159         elif self.tritonserver_type == "grpc":
160             with reserve_free_port(host="0.0.0.0", enable_so_reuseport=True) as port:
161                 pass
162             return f"0.0.0.0:{port}"
163         else:
164             raise ValueError(f"Invalid Triton Server type: {self.tritonserver_type}")
165 
166     def init_local(self, quiet: bool = False) -> None:
167         _logger.warning(
168             "TritonRunner '%s' will not be available for development mode.", self.name
169         )
170 
171     def init_client(
172         self,
173         handle_class: type[RunnerHandle] | None = None,
174         *args: t.Any,
175         **kwargs: t.Any,
176     ):
177         from ._internal.runner.runner_handle.remote import TritonRunnerHandle
178 
179         if handle_class is None:
180             handle_class = TritonRunnerHandle
181 
182         super().init_client(handle_class=handle_class, *args, **kwargs)
183 
184     def destroy(self):
185         _object_setattr(self, "_runner_handle", _DummyRunnerHandle())
186 
187     # Even though the below overload overlaps, it is ok to ignore the warning since types
188     # for TritonRunner can handle both function from client and LiteralString from model name.
189     @t.overload
190     def __getattr__(self, item: t.Literal["__attrs_init__"]) -> t.Callable[..., None]:  # type: ignore (overload warning)
191         ...
192 
193     @t.overload
194     def __getattr__(
195         self, item: _ClientMethod
196     ) -> t.Callable[..., t.Coroutine[t.Any, t.Any, t.Any]]:
197         ...
198 
199     @t.overload
200     def __getattr__(
201         self, item: _ModelName
202     ) -> _RunnerMethod[
203         t.Any, _P, _tritongrpcclient.InferResult | _tritonhttpclient.InferResult
204     ]:
205         ...
206 
207     def __getattr__(self, item: str) -> t.Any:
208         from ._internal.runner.runner_handle.remote import TritonRunnerHandle
209 
210         if isinstance(self._runner_handle, TritonRunnerHandle):
211             if item in self._runner_handle.client_methods:
212                 # NOTE: auto wrap triton methods to its respective clients
213                 if self.tritonserver_type == "grpc":
214                     return _handle_triton_exception(
215                         getattr(self._runner_handle.grpc_client, item)
216                     )
217                 else:
218                     return _handle_triton_exception(
219                         getattr(self._runner_handle.http_client, item)
220                     )
221             else:
222                 # if given item is not a client method, then we assume it is a model name.
223                 # Hence, we will return a RunnerMethod that will be responsible for this model handle.
224                 RT = (
225                     _tritonhttpclient.InferResult
226                     if self.tritonserver_type == "http"
227                     else _tritongrpcclient.InferResult
228                 )
229                 return _RunnerMethod[t.Any, _P, RT](
230                     runner=self,
231                     name=item,
232                     config=_RunnableMethodConfig(batchable=True, batch_dim=(0, 0)),
233                     max_batch_size=0,
234                     max_latency_ms=10000,
235                 )
236 
237         return super().__getattribute__(item)
238 
239 
240 Runner = _TritonRunner
241
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/bentoml/triton.py b/src/bentoml/triton.py
--- a/src/bentoml/triton.py
+++ b/src/bentoml/triton.py
@@ -20,6 +20,7 @@
     handle_triton_exception as _handle_triton_exception,
 )
 from ._internal.utils import LazyLoader as _LazyLoader
+from .exceptions import StateException as _StateException
 
 if t.TYPE_CHECKING:
     import tritonclient.grpc.aio as _tritongrpcclient
@@ -168,18 +169,27 @@
             "TritonRunner '%s' will not be available for development mode.", self.name
         )
 
+    def _set_handle(
+        self, handle_class: type[RunnerHandle], *args: t.Any, **kwargs: t.Any
+    ) -> None:
+        if not isinstance(self._runner_handle, _DummyRunnerHandle):
+            raise _StateException("Runner already initialized")
+
+        runner_handle = handle_class(self, *args, **kwargs)
+        _object_setattr(self, "_runner_handle", runner_handle)
+
     def init_client(
         self,
         handle_class: type[RunnerHandle] | None = None,
         *args: t.Any,
         **kwargs: t.Any,
     ):
-        from ._internal.runner.runner_handle.remote import TritonRunnerHandle
-
         if handle_class is None:
-            handle_class = TritonRunnerHandle
+            from ._internal.runner.runner_handle.remote import TritonRunnerHandle
 
-        super().init_client(handle_class=handle_class, *args, **kwargs)
+            self._set_handle(TritonRunnerHandle)
+        else:
+            self._set_handle(handle_class, *args, **kwargs)
 
     def destroy(self):
         _object_setattr(self, "_runner_handle", _DummyRunnerHandle())

{"golden_diff": "diff --git a/src/bentoml/triton.py b/src/bentoml/triton.py\n--- a/src/bentoml/triton.py\n+++ b/src/bentoml/triton.py\n@@ -20,6 +20,7 @@\n     handle_triton_exception as _handle_triton_exception,\n )\n from ._internal.utils import LazyLoader as _LazyLoader\n+from .exceptions import StateException as _StateException\n \n if t.TYPE_CHECKING:\n     import tritonclient.grpc.aio as _tritongrpcclient\n@@ -168,18 +169,27 @@\n             \"TritonRunner '%s' will not be available for development mode.\", self.name\n         )\n \n+    def _set_handle(\n+        self, handle_class: type[RunnerHandle], *args: t.Any, **kwargs: t.Any\n+    ) -> None:\n+        if not isinstance(self._runner_handle, _DummyRunnerHandle):\n+            raise _StateException(\"Runner already initialized\")\n+\n+        runner_handle = handle_class(self, *args, **kwargs)\n+        _object_setattr(self, \"_runner_handle\", runner_handle)\n+\n     def init_client(\n         self,\n         handle_class: type[RunnerHandle] | None = None,\n         *args: t.Any,\n         **kwargs: t.Any,\n     ):\n-        from ._internal.runner.runner_handle.remote import TritonRunnerHandle\n-\n         if handle_class is None:\n-            handle_class = TritonRunnerHandle\n+            from ._internal.runner.runner_handle.remote import TritonRunnerHandle\n \n-        super().init_client(handle_class=handle_class, *args, **kwargs)\n+            self._set_handle(TritonRunnerHandle)\n+        else:\n+            self._set_handle(handle_class, *args, **kwargs)\n \n     def destroy(self):\n         _object_setattr(self, \"_runner_handle\", _DummyRunnerHandle())\n", "issue": "bug: Triton Integration failed with an AttributeError\n### Describe the bug\n\nI try to integrate Triton Inference Server into BentoML and I follow both the official documentation(https://docs.bentoml.org/en/latest/integrations/triton.html) and the following example(https://github.com/bentoml/BentoML/tree/main/examples/triton/onnx). However, I get an AttributeError with the following message:\r\n\" '_TritonRunner' object has no attribute 'onnx_fp_16'\", where onnx_fp_16 is my model deployed in Triton.\r\nThe code in my service.py file is:\r\n```\r\nimport bentoml\r\nfrom bentoml.io import JSON\r\nfrom src.tokenizer import RDATokenizer\r\nfrom src.request import RDARequest\r\nfrom src.response import RDAResponse\r\nfrom src.config import  load_config\r\nfrom src.utils.model import load_topics_and_mapping, get_multi_label_binarizer, post_predict, load_taxonomy_root,\\\r\n    load_parent_child_relationship\r\n\r\nconfigs = load_config()\r\nrda_triton_runner = bentoml.triton.Runner(\"rdav2\",\r\n                                          model_repository=\"src/model_repository\",\r\n                                          cli_args=[\"--model-control-mode=explicit\",\r\n                                                    \"--load-model=onnx_fp_16\",\r\n                                                    \"--log-verbose=1\"])\r\n\r\nsvc = bentoml.Service(\"rdav2\", runners=[rda_triton_runner])\r\ntokenizer = RDATokenizer(tokenizer_path=configs.tokenizer_path)\r\nlabels_binarizer = get_multi_label_binarizer(config=configs)\r\ntopics, topics_mapping = load_topics_and_mapping(config=configs)\r\ntax_root = load_taxonomy_root(configs.tax_root_path)\r\nparent_child = load_parent_child_relationship(config=configs)\r\nid_to_fos = {id_: fos for fos, id_ in topics_mapping.items()}\r\n\r\n\r\n@svc.api(input=JSON(pydantic_model=RDARequest), output=JSON(pydantic_model=RDAResponse))\r\nasync def predict(rda_request: RDARequest) -> RDAResponse:\r\n    \"\"\"\r\n    The input will be the title, abstract, k and, threshold and the output will be the prediction of the model\r\n    :param rda_request: the request object\r\n    :return: a dictionary where the key is the metadata name and the value is the value of the metadata.\r\n    the metadata will be a list with the tags, the tags ids, the probabilities, the ancestros and the ancestors id\r\n    \"\"\"\r\n    title = rda_request.title\r\n    abstract = rda_request.abstract\r\n    k = rda_request.k\r\n    threshold = rda_request.threshold\r\n    text = title + \". \" + abstract\r\n    encoded_text = await tokenizer.tokenize(text=text)\r\n    logits = await rda_triton_runner.onnx_fp_16.async_run(encoded_text)\r\n    logits = logits[0].squeeze()\r\n    response = await post_predict(logits=logits,\r\n                                  labels_binarizer=labels_binarizer,\r\n                                  k=k,\r\n                                  threshold=threshold,\r\n                                  topics=topics,\r\n                                  topics_mapping=topics_mapping,\r\n                                  tax_roots=tax_root,\r\n                                  parent_child=parent_child,\r\n                                  id_to_fos=id_to_fos)\r\n    return response\r\n```\r\nMy _bentofile.yaml_ is the following:\r\n```\r\nservice: \"service:svc\"\r\ninclude:\r\n- \"*.py\"\r\n- \"/model_repository\"\r\n- \"/configs\"\r\n- \"/checkpoints\"\r\npython:\r\n  requirements_txt: \"requirements.txt\"\r\ndocker:\r\n  base_image: nvcr.io/nvidia/tritonserver:22.12-py3\r\n```\r\nI tried to run the service with a container and the commands I typed to do so were the followings:\r\n- bentoml build --version 0.0.2\r\n- bentoml containerize rdav2:0.0.2\r\n- docker run -it --rm -p 3000:3000 rdav2:0.0.2 serve --production\r\n\r\nI guess there is something wrong in __rda_triton_runner__ object, because when I debug the service, I get an empty list in the __models__ parameter. I am not sure about that, and that's the reason I opened this issue.\n\n### To reproduce\n\n_No response_\n\n### Expected behavior\n\n_No response_\n\n### Environment\n\nbentoml[triton]: 1.0.32\r\npython: 3.8\r\nplatform: Ubuntu: 20.10\n", "before_files": [{"content": "from __future__ import annotations\n\nimport logging\nimport typing as t\nfrom functools import cached_property\n\nimport attr\nfrom simple_di import Provide as _Provide\nfrom simple_di import inject as _inject\n\nfrom ._internal.configuration import get_debug_mode as _get_debug_mode\nfrom ._internal.configuration.containers import BentoMLContainer as _BentoMLContainer\nfrom ._internal.runner.runnable import RunnableMethodConfig as _RunnableMethodConfig\nfrom ._internal.runner.runner import AbstractRunner as _AbstractRunner\nfrom ._internal.runner.runner import RunnerMethod as _RunnerMethod\nfrom ._internal.runner.runner import object_setattr as _object_setattr\nfrom ._internal.runner.runner_handle import DummyRunnerHandle as _DummyRunnerHandle\nfrom ._internal.runner.runner_handle.remote import TRITON_EXC_MSG as _TRITON_EXC_MSG\nfrom ._internal.runner.runner_handle.remote import (\n    handle_triton_exception as _handle_triton_exception,\n)\nfrom ._internal.utils import LazyLoader as _LazyLoader\n\nif t.TYPE_CHECKING:\n    import tritonclient.grpc.aio as _tritongrpcclient\n    import tritonclient.http.aio as _tritonhttpclient\n\n    from ._internal.runner.runner_handle import RunnerHandle\n\n    _P = t.ParamSpec(\"_P\")\n\n    _LogFormat = t.Literal[\"default\", \"ISO8601\"]\n    _GrpcInferResponseCompressionLevel = t.Literal[\"none\", \"low\", \"medium\", \"high\"]\n    _TraceLevel = t.Literal[\"OFF\", \"TIMESTAMPS\", \"TENSORS\"]\n    _RateLimit = t.Literal[\"execution_count\", \"off\"]\n    _TritonServerType = t.Literal[\"grpc\", \"http\"]\n\n    _ClientMethod = t.Literal[\n        \"get_cuda_shared_memory_status\",\n        \"get_inference_statistics\",\n        \"get_log_settings\",\n        \"get_model_config\",\n        \"get_model_metadata\",\n        \"get_model_repository_index\",\n        \"get_server_metadata\",\n        \"get_system_shared_memory_status\",\n        \"get_trace_settings\",\n        \"infer\",\n        \"is_model_ready\",\n        \"is_server_live\",\n        \"is_server_ready\",\n        \"load_model\",\n        \"register_cuda_shared_memory\",\n        \"register_system_shared_memory\",\n        \"stream_infer\",\n        \"unload_model\",\n        \"unregister_cuda_shared_memory\",\n        \"unregister_system_shared_memory\",\n        \"update_log_settings\",\n        \"update_trace_settings\",\n    ]\n    _ModelName = t.Annotated[str, t.LiteralString]\n\nelse:\n    _P = t.TypeVar(\"_P\")\n\n    _LogFormat = _GrpcInferResponseCompressionLevel = _TraceLevel = _RateLimit = str\n\n    _tritongrpcclient = _LazyLoader(\n        \"_tritongrpcclient\", globals(), \"tritonclient.grpc.aio\", exc_msg=_TRITON_EXC_MSG\n    )\n    _tritonhttpclient = _LazyLoader(\n        \"_tritonhttpclient\", globals(), \"tritonclient.http.aio\", exc_msg=_TRITON_EXC_MSG\n    )\n\n_logger = logging.getLogger(__name__)\n\n__all__ = [\"Runner\"]\n\n\n@attr.define(slots=False, frozen=True, eq=False)\nclass _TritonRunner(_AbstractRunner):\n    repository_path: str\n\n    tritonserver_type: _TritonServerType = attr.field(\n        default=\"grpc\", validator=attr.validators.in_([\"grpc\", \"http\"])\n    )\n    cli_args: list[str] = attr.field(factory=list)\n\n    _runner_handle: RunnerHandle = attr.field(init=False, factory=_DummyRunnerHandle)\n\n    @_inject\n    async def runner_handle_is_ready(\n        self,\n        timeout: int = _Provide[\n            _BentoMLContainer.api_server_config.runner_probe.timeout\n        ],\n    ) -> bool:\n        \"\"\"\n        Check if given runner handle is ready. This will be used as readiness probe in Kubernetes.\n        \"\"\"\n        return await self._runner_handle.is_ready(timeout)\n\n    def __init__(\n        self,\n        name: str,\n        model_repository: str,\n        tritonserver_type: _TritonServerType = \"grpc\",\n        cli_args: list[str] | None = None,\n    ):\n        if cli_args is None:\n            cli_args = []\n\n        cli_args.append(f\"--model-repository={model_repository}\")\n\n        if tritonserver_type == \"http\":\n            cli_args.extend(\n                [\n                    \"--allow-grpc=False\",\n                    \"--http-address=127.0.0.1\",\n                ]\n            )\n        elif tritonserver_type == \"grpc\":\n            cli_args.extend(\n                [\n                    \"--reuse-grpc-port=1\",\n                    \"--allow-http=False\",\n                    \"--grpc-address=0.0.0.0\",\n                ]\n            )\n\n        # default settings, disable metrics\n        cli_args.extend([f\"--log-verbose={1 if _get_debug_mode() else 0}\"])\n\n        if not all(s.startswith(\"--\") for s in cli_args):\n            raise ValueError(\n                \"cli_args should be a list of strings starting with '--' for TritonRunner.\"\n            )\n\n        self.__attrs_init__(\n            name=name,\n            models=None,\n            resource_config=None,\n            runnable_class=self.__class__,\n            repository_path=model_repository,\n            tritonserver_type=tritonserver_type,\n            cli_args=cli_args,\n            embedded=False,  # NOTE: TritonRunner shouldn't be used as embedded.\n        )\n\n    @cached_property\n    def protocol_address(self):\n        from ._internal.utils import reserve_free_port\n\n        if self.tritonserver_type == \"http\":\n            with reserve_free_port(host=\"127.0.0.1\") as port:\n                pass\n            return f\"127.0.0.1:{port}\"\n        elif self.tritonserver_type == \"grpc\":\n            with reserve_free_port(host=\"0.0.0.0\", enable_so_reuseport=True) as port:\n                pass\n            return f\"0.0.0.0:{port}\"\n        else:\n            raise ValueError(f\"Invalid Triton Server type: {self.tritonserver_type}\")\n\n    def init_local(self, quiet: bool = False) -> None:\n        _logger.warning(\n            \"TritonRunner '%s' will not be available for development mode.\", self.name\n        )\n\n    def init_client(\n        self,\n        handle_class: type[RunnerHandle] | None = None,\n        *args: t.Any,\n        **kwargs: t.Any,\n    ):\n        from ._internal.runner.runner_handle.remote import TritonRunnerHandle\n\n        if handle_class is None:\n            handle_class = TritonRunnerHandle\n\n        super().init_client(handle_class=handle_class, *args, **kwargs)\n\n    def destroy(self):\n        _object_setattr(self, \"_runner_handle\", _DummyRunnerHandle())\n\n    # Even though the below overload overlaps, it is ok to ignore the warning since types\n    # for TritonRunner can handle both function from client and LiteralString from model name.\n    @t.overload\n    def __getattr__(self, item: t.Literal[\"__attrs_init__\"]) -> t.Callable[..., None]:  # type: ignore (overload warning)\n        ...\n\n    @t.overload\n    def __getattr__(\n        self, item: _ClientMethod\n    ) -> t.Callable[..., t.Coroutine[t.Any, t.Any, t.Any]]:\n        ...\n\n    @t.overload\n    def __getattr__(\n        self, item: _ModelName\n    ) -> _RunnerMethod[\n        t.Any, _P, _tritongrpcclient.InferResult | _tritonhttpclient.InferResult\n    ]:\n        ...\n\n    def __getattr__(self, item: str) -> t.Any:\n        from ._internal.runner.runner_handle.remote import TritonRunnerHandle\n\n        if isinstance(self._runner_handle, TritonRunnerHandle):\n            if item in self._runner_handle.client_methods:\n                # NOTE: auto wrap triton methods to its respective clients\n                if self.tritonserver_type == \"grpc\":\n                    return _handle_triton_exception(\n                        getattr(self._runner_handle.grpc_client, item)\n                    )\n                else:\n                    return _handle_triton_exception(\n                        getattr(self._runner_handle.http_client, item)\n                    )\n            else:\n                # if given item is not a client method, then we assume it is a model name.\n                # Hence, we will return a RunnerMethod that will be responsible for this model handle.\n                RT = (\n                    _tritonhttpclient.InferResult\n                    if self.tritonserver_type == \"http\"\n                    else _tritongrpcclient.InferResult\n                )\n                return _RunnerMethod[t.Any, _P, RT](\n                    runner=self,\n                    name=item,\n                    config=_RunnableMethodConfig(batchable=True, batch_dim=(0, 0)),\n                    max_batch_size=0,\n                    max_latency_ms=10000,\n                )\n\n        return super().__getattribute__(item)\n\n\nRunner = _TritonRunner\n", "path": "src/bentoml/triton.py"}], "after_files": [{"content": "from __future__ import annotations\n\nimport logging\nimport typing as t\nfrom functools import cached_property\n\nimport attr\nfrom simple_di import Provide as _Provide\nfrom simple_di import inject as _inject\n\nfrom ._internal.configuration import get_debug_mode as _get_debug_mode\nfrom ._internal.configuration.containers import BentoMLContainer as _BentoMLContainer\nfrom ._internal.runner.runnable import RunnableMethodConfig as _RunnableMethodConfig\nfrom ._internal.runner.runner import AbstractRunner as _AbstractRunner\nfrom ._internal.runner.runner import RunnerMethod as _RunnerMethod\nfrom ._internal.runner.runner import object_setattr as _object_setattr\nfrom ._internal.runner.runner_handle import DummyRunnerHandle as _DummyRunnerHandle\nfrom ._internal.runner.runner_handle.remote import TRITON_EXC_MSG as _TRITON_EXC_MSG\nfrom ._internal.runner.runner_handle.remote import (\n    handle_triton_exception as _handle_triton_exception,\n)\nfrom ._internal.utils import LazyLoader as _LazyLoader\nfrom .exceptions import StateException as _StateException\n\nif t.TYPE_CHECKING:\n    import tritonclient.grpc.aio as _tritongrpcclient\n    import tritonclient.http.aio as _tritonhttpclient\n\n    from ._internal.runner.runner_handle import RunnerHandle\n\n    _P = t.ParamSpec(\"_P\")\n\n    _LogFormat = t.Literal[\"default\", \"ISO8601\"]\n    _GrpcInferResponseCompressionLevel = t.Literal[\"none\", \"low\", \"medium\", \"high\"]\n    _TraceLevel = t.Literal[\"OFF\", \"TIMESTAMPS\", \"TENSORS\"]\n    _RateLimit = t.Literal[\"execution_count\", \"off\"]\n    _TritonServerType = t.Literal[\"grpc\", \"http\"]\n\n    _ClientMethod = t.Literal[\n        \"get_cuda_shared_memory_status\",\n        \"get_inference_statistics\",\n        \"get_log_settings\",\n        \"get_model_config\",\n        \"get_model_metadata\",\n        \"get_model_repository_index\",\n        \"get_server_metadata\",\n        \"get_system_shared_memory_status\",\n        \"get_trace_settings\",\n        \"infer\",\n        \"is_model_ready\",\n        \"is_server_live\",\n        \"is_server_ready\",\n        \"load_model\",\n        \"register_cuda_shared_memory\",\n        \"register_system_shared_memory\",\n        \"stream_infer\",\n        \"unload_model\",\n        \"unregister_cuda_shared_memory\",\n        \"unregister_system_shared_memory\",\n        \"update_log_settings\",\n        \"update_trace_settings\",\n    ]\n    _ModelName = t.Annotated[str, t.LiteralString]\n\nelse:\n    _P = t.TypeVar(\"_P\")\n\n    _LogFormat = _GrpcInferResponseCompressionLevel = _TraceLevel = _RateLimit = str\n\n    _tritongrpcclient = _LazyLoader(\n        \"_tritongrpcclient\", globals(), \"tritonclient.grpc.aio\", exc_msg=_TRITON_EXC_MSG\n    )\n    _tritonhttpclient = _LazyLoader(\n        \"_tritonhttpclient\", globals(), \"tritonclient.http.aio\", exc_msg=_TRITON_EXC_MSG\n    )\n\n_logger = logging.getLogger(__name__)\n\n__all__ = [\"Runner\"]\n\n\n@attr.define(slots=False, frozen=True, eq=False)\nclass _TritonRunner(_AbstractRunner):\n    repository_path: str\n\n    tritonserver_type: _TritonServerType = attr.field(\n        default=\"grpc\", validator=attr.validators.in_([\"grpc\", \"http\"])\n    )\n    cli_args: list[str] = attr.field(factory=list)\n\n    _runner_handle: RunnerHandle = attr.field(init=False, factory=_DummyRunnerHandle)\n\n    @_inject\n    async def runner_handle_is_ready(\n        self,\n        timeout: int = _Provide[\n            _BentoMLContainer.api_server_config.runner_probe.timeout\n        ],\n    ) -> bool:\n        \"\"\"\n        Check if given runner handle is ready. This will be used as readiness probe in Kubernetes.\n        \"\"\"\n        return await self._runner_handle.is_ready(timeout)\n\n    def __init__(\n        self,\n        name: str,\n        model_repository: str,\n        tritonserver_type: _TritonServerType = \"grpc\",\n        cli_args: list[str] | None = None,\n    ):\n        if cli_args is None:\n            cli_args = []\n\n        cli_args.append(f\"--model-repository={model_repository}\")\n\n        if tritonserver_type == \"http\":\n            cli_args.extend(\n                [\n                    \"--allow-grpc=False\",\n                    \"--http-address=127.0.0.1\",\n                ]\n            )\n        elif tritonserver_type == \"grpc\":\n            cli_args.extend(\n                [\n                    \"--reuse-grpc-port=1\",\n                    \"--allow-http=False\",\n                    \"--grpc-address=0.0.0.0\",\n                ]\n            )\n\n        # default settings, disable metrics\n        cli_args.extend([f\"--log-verbose={1 if _get_debug_mode() else 0}\"])\n\n        if not all(s.startswith(\"--\") for s in cli_args):\n            raise ValueError(\n                \"cli_args should be a list of strings starting with '--' for TritonRunner.\"\n            )\n\n        self.__attrs_init__(\n            name=name,\n            models=None,\n            resource_config=None,\n            runnable_class=self.__class__,\n            repository_path=model_repository,\n            tritonserver_type=tritonserver_type,\n            cli_args=cli_args,\n            embedded=False,  # NOTE: TritonRunner shouldn't be used as embedded.\n        )\n\n    @cached_property\n    def protocol_address(self):\n        from ._internal.utils import reserve_free_port\n\n        if self.tritonserver_type == \"http\":\n            with reserve_free_port(host=\"127.0.0.1\") as port:\n                pass\n            return f\"127.0.0.1:{port}\"\n        elif self.tritonserver_type == \"grpc\":\n            with reserve_free_port(host=\"0.0.0.0\", enable_so_reuseport=True) as port:\n                pass\n            return f\"0.0.0.0:{port}\"\n        else:\n            raise ValueError(f\"Invalid Triton Server type: {self.tritonserver_type}\")\n\n    def init_local(self, quiet: bool = False) -> None:\n        _logger.warning(\n            \"TritonRunner '%s' will not be available for development mode.\", self.name\n        )\n\n    def _set_handle(\n        self, handle_class: type[RunnerHandle], *args: t.Any, **kwargs: t.Any\n    ) -> None:\n        if not isinstance(self._runner_handle, _DummyRunnerHandle):\n            raise _StateException(\"Runner already initialized\")\n\n        runner_handle = handle_class(self, *args, **kwargs)\n        _object_setattr(self, \"_runner_handle\", runner_handle)\n\n    def init_client(\n        self,\n        handle_class: type[RunnerHandle] | None = None,\n        *args: t.Any,\n        **kwargs: t.Any,\n    ):\n        if handle_class is None:\n            from ._internal.runner.runner_handle.remote import TritonRunnerHandle\n\n            self._set_handle(TritonRunnerHandle)\n        else:\n            self._set_handle(handle_class, *args, **kwargs)\n\n    def destroy(self):\n        _object_setattr(self, \"_runner_handle\", _DummyRunnerHandle())\n\n    # Even though the below overload overlaps, it is ok to ignore the warning since types\n    # for TritonRunner can handle both function from client and LiteralString from model name.\n    @t.overload\n    def __getattr__(self, item: t.Literal[\"__attrs_init__\"]) -> t.Callable[..., None]:  # type: ignore (overload warning)\n        ...\n\n    @t.overload\n    def __getattr__(\n        self, item: _ClientMethod\n    ) -> t.Callable[..., t.Coroutine[t.Any, t.Any, t.Any]]:\n        ...\n\n    @t.overload\n    def __getattr__(\n        self, item: _ModelName\n    ) -> _RunnerMethod[\n        t.Any, _P, _tritongrpcclient.InferResult | _tritonhttpclient.InferResult\n    ]:\n        ...\n\n    def __getattr__(self, item: str) -> t.Any:\n        from ._internal.runner.runner_handle.remote import TritonRunnerHandle\n\n        if isinstance(self._runner_handle, TritonRunnerHandle):\n            if item in self._runner_handle.client_methods:\n                # NOTE: auto wrap triton methods to its respective clients\n                if self.tritonserver_type == \"grpc\":\n                    return _handle_triton_exception(\n                        getattr(self._runner_handle.grpc_client, item)\n                    )\n                else:\n                    return _handle_triton_exception(\n                        getattr(self._runner_handle.http_client, item)\n                    )\n            else:\n                # if given item is not a client method, then we assume it is a model name.\n                # Hence, we will return a RunnerMethod that will be responsible for this model handle.\n                RT = (\n                    _tritonhttpclient.InferResult\n                    if self.tritonserver_type == \"http\"\n                    else _tritongrpcclient.InferResult\n                )\n                return _RunnerMethod[t.Any, _P, RT](\n                    runner=self,\n                    name=item,\n                    config=_RunnableMethodConfig(batchable=True, batch_dim=(0, 0)),\n                    max_batch_size=0,\n                    max_latency_ms=10000,\n                )\n\n        return super().__getattribute__(item)\n\n\nRunner = _TritonRunner\n", "path": "src/bentoml/triton.py"}]}