We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Kserve ModelServer().start(list_of_models) fails on Windows
/kind bug

**What steps did you take and what happened:**
When ModelServer().start(list_of_models) is called, exception is raised at this location: https://github.com/kserve/kserve/blob/0ba642f9b6fa40b38fe272d04c8dfde3598bfb05/python/kserve/kserve/model_server.py#L154

**What did you expect to happen:**

This simple workaround makes it possible to run the code on Windows:
```
            # SIGQUIT is not available on Windows
            try:
                sig_list = [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]
            except:
                sig_list = [signal.SIGINT, signal.SIGTERM]

            # add_signal_handler may not have been implemented
            try:
                for sig in sig_list:
                    loop.add_signal_handler(
                        sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))
                    )
            except:
                pass
```

Even though Windows-based containers are very unlikely to be used for deploying KServe inference services, the ability to run kserve-based code locally on Windows-based machines for development/debugging purposes often comes very handy. I suggest fixing the Windows incompatibility issue in the next release of the kserve Python package.

**Environment:**

- OS (e.g. from `/etc/os-release`): Windows 10


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `python/kserve/kserve/model_server.py`
Content:
```
1 # Copyright 2022 The KServe Authors.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #    http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import argparse
16 import asyncio
17 import concurrent.futures
18 import multiprocessing
19 import signal
20 import socket
21 from multiprocessing import Process
22 from typing import Dict, List, Optional, Union
23 
24 from ray import serve as rayserve
25 from ray.serve.api import Deployment
26 from ray.serve.handle import RayServeHandle
27 
28 from .logging import KSERVE_LOG_CONFIG, logger
29 from .model import Model
30 from .model_repository import ModelRepository
31 from .protocol.dataplane import DataPlane
32 from .protocol.grpc.server import GRPCServer
33 from .protocol.model_repository_extension import ModelRepositoryExtension
34 from .protocol.rest.server import UvicornServer
35 from .utils import utils
36 
37 DEFAULT_HTTP_PORT = 8080
38 DEFAULT_GRPC_PORT = 8081
39 
40 parser = argparse.ArgumentParser(add_help=False)
41 parser.add_argument("--http_port", default=DEFAULT_HTTP_PORT, type=int,
42                     help="The HTTP Port listened to by the model server.")
43 parser.add_argument("--grpc_port", default=DEFAULT_GRPC_PORT, type=int,
44                     help="The GRPC Port listened to by the model server.")
45 parser.add_argument("--workers", default=1, type=int,
46                     help="The number of workers for multi-processing.")
47 parser.add_argument("--max_threads", default=4, type=int,
48                     help="The number of max processing threads in each worker.")
49 parser.add_argument('--max_asyncio_workers', default=None, type=int,
50                     help='Max number of asyncio workers to spawn')
51 parser.add_argument("--enable_grpc", default=True, type=lambda x: utils.strtobool(x),
52                     help="Enable gRPC for the model server")
53 parser.add_argument("--enable_docs_url", default=False, type=lambda x: utils.strtobool(x),
54                     help="Enable docs url '/docs' to display Swagger UI.")
55 parser.add_argument("--enable_latency_logging", default=True, type=lambda x: utils.strtobool(x),
56                     help="Output a log per request with latency metrics.")
57 parser.add_argument("--configure_logging", default=True, type=lambda x: utils.strtobool(x),
58                     help="Whether to configure KServe and Uvicorn logging")
59 parser.add_argument("--log_config_file", default=None, type=str,
60                     help="File path containing UvicornServer's log config. Needs to be a yaml or json file.")
61 parser.add_argument("--access_log_format", default=None, type=str,
62                     help="Format to set for the access log (provided by asgi-logger).")
63 
64 args, _ = parser.parse_known_args()
65 
66 
67 class ModelServer:
68     """KServe ModelServer
69 
70     Args:
71         http_port (int): HTTP port. Default: ``8080``.
72         grpc_port (int): GRPC port. Default: ``8081``.
73         workers (int): Number of workers for uvicorn. Default: ``1``.
74         max_threads (int): Max number of processing threads. Default: ``4``
75         max_asyncio_workers (int): Max number of AsyncIO threads. Default: ``None``
76         registered_models (ModelRepository): Model repository with registered models.
77         enable_grpc (bool): Whether to turn on grpc server. Default: ``True``
78         enable_docs_url (bool): Whether to turn on ``/docs`` Swagger UI. Default: ``False``.
79         enable_latency_logging (bool): Whether to log latency metric. Default: ``True``.
80         configure_logging (bool): Whether to configure KServe and Uvicorn logging. Default: ``True``.
81         log_config (dict or str): File path or dict containing log config. Default: ``None``.
82         access_log_format (string): Format to set for the access log (provided by asgi-logger). Default: ``None``
83     """
84 
85     def __init__(self, http_port: int = args.http_port,
86                  grpc_port: int = args.grpc_port,
87                  workers: int = args.workers,
88                  max_threads: int = args.max_threads,
89                  max_asyncio_workers: int = args.max_asyncio_workers,
90                  registered_models: ModelRepository = ModelRepository(),
91                  enable_grpc: bool = args.enable_grpc,
92                  enable_docs_url: bool = args.enable_docs_url,
93                  enable_latency_logging: bool = args.enable_latency_logging,
94                  configure_logging: bool = args.configure_logging,
95                  log_config: Optional[Union[Dict, str]] = args.log_config_file,
96                  access_log_format: str = args.access_log_format):
97         self.registered_models = registered_models
98         self.http_port = http_port
99         self.grpc_port = grpc_port
100         self.workers = workers
101         self.max_threads = max_threads
102         self.max_asyncio_workers = max_asyncio_workers
103         self.enable_grpc = enable_grpc
104         self.enable_docs_url = enable_docs_url
105         self.enable_latency_logging = enable_latency_logging
106         self.dataplane = DataPlane(model_registry=registered_models)
107         self.model_repository_extension = ModelRepositoryExtension(
108             model_registry=self.registered_models)
109         self._grpc_server = None
110         if self.enable_grpc:
111             self._grpc_server = GRPCServer(grpc_port, self.dataplane,
112                                            self.model_repository_extension)
113 
114         # Logs can be passed as a path to a file or a dictConfig.
115         # We rely on Uvicorn to configure the loggers for us.
116         if configure_logging:
117             self.log_config = log_config if log_config is not None else KSERVE_LOG_CONFIG
118         else:
119             # By setting log_config to None we tell Uvicorn not to configure logging
120             self.log_config = None
121 
122         self.access_log_format = access_log_format
123 
124     def start(self, models: Union[List[Model], Dict[str, Deployment]]) -> None:
125         if isinstance(models, list):
126             for model in models:
127                 if isinstance(model, Model):
128                     self.register_model(model)
129                     # pass whether to log request latency into the model
130                     model.enable_latency_logging = self.enable_latency_logging
131                 else:
132                     raise RuntimeError("Model type should be 'Model'")
133         elif isinstance(models, dict):
134             if all([isinstance(v, Deployment) for v in models.values()]):
135                 # TODO: make this port number a variable
136                 rayserve.start(detached=True, http_options={"host": "0.0.0.0", "port": 9071})
137                 for key in models:
138                     models[key].deploy()
139                     handle = models[key].get_handle()
140                     self.register_model_handle(key, handle)
141             else:
142                 raise RuntimeError("Model type should be RayServe Deployment")
143         else:
144             raise RuntimeError("Unknown model collection types")
145 
146         if self.max_asyncio_workers is None:
147             # formula as suggest in https://bugs.python.org/issue35279
148             self.max_asyncio_workers = min(32, utils.cpu_count() + 4)
149         logger.info(f"Setting max asyncio worker threads as {self.max_asyncio_workers}")
150         asyncio.get_event_loop().set_default_executor(
151             concurrent.futures.ThreadPoolExecutor(max_workers=self.max_asyncio_workers))
152 
153         async def serve():
154             logger.info(f"Starting uvicorn with {self.workers} workers")
155             loop = asyncio.get_event_loop()
156             for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:
157                 loop.add_signal_handler(
158                     sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))
159                 )
160             if self.workers == 1:
161                 self._rest_server = UvicornServer(self.http_port, [],
162                                                   self.dataplane, self.model_repository_extension,
163                                                   self.enable_docs_url,
164                                                   log_config=self.log_config,
165                                                   access_log_format=self.access_log_format)
166                 await self._rest_server.run()
167             else:
168                 # Since py38 MacOS/Windows defaults to use spawn for starting multiprocessing.
169                 # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
170                 # Spawn does not work with FastAPI/uvicorn in multiprocessing mode, use fork for multiprocessing
171                 # https://github.com/tiangolo/fastapi/issues/1586
172                 serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
173                 serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
174                 serversocket.bind(('0.0.0.0', self.http_port))
175                 serversocket.listen(5)
176                 multiprocessing.set_start_method('fork')
177                 server = UvicornServer(self.http_port, [serversocket],
178                                        self.dataplane, self.model_repository_extension,
179                                        self.enable_docs_url, log_config=self.log_config,
180                                        access_log_format=self.access_log_format)
181                 for _ in range(self.workers):
182                     p = Process(target=server.run_sync)
183                     p.start()
184 
185         async def servers_task():
186             servers = [serve()]
187             if self.enable_grpc:
188                 servers.append(self._grpc_server.start(self.max_threads))
189             await asyncio.gather(*servers)
190 
191         asyncio.run(servers_task())
192 
193     async def stop(self, sig: Optional[int] = None):
194         logger.info("Stopping the model server")
195         if self._rest_server:
196             logger.info("Stopping the rest server")
197             await self._rest_server.stop()
198         if self._grpc_server:
199             logger.info("Stopping the grpc server")
200             await self._grpc_server.stop(sig)
201 
202     def register_model_handle(self, name: str, model_handle: RayServeHandle):
203         self.registered_models.update_handle(name, model_handle)
204         logger.info("Registering model handle: %s", name)
205 
206     def register_model(self, model: Model):
207         if not model.name:
208             raise Exception(
209                 "Failed to register model, model.name must be provided.")
210         self.registered_models.update(model)
211         logger.info("Registering model: %s", model.name)
212
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/python/kserve/kserve/model_server.py b/python/kserve/kserve/model_server.py
--- a/python/kserve/kserve/model_server.py
+++ b/python/kserve/kserve/model_server.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import argparse
 import asyncio
 import concurrent.futures
@@ -153,7 +154,12 @@
         async def serve():
             logger.info(f"Starting uvicorn with {self.workers} workers")
             loop = asyncio.get_event_loop()
-            for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:
+            if sys.platform not in ['win32', 'win64']:
+                sig_list = [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]
+            else:
+                sig_list = [signal.SIGINT, signal.SIGTERM]
+
+            for sig in sig_list:
                 loop.add_signal_handler(
                     sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))
                 )

{"golden_diff": "diff --git a/python/kserve/kserve/model_server.py b/python/kserve/kserve/model_server.py\n--- a/python/kserve/kserve/model_server.py\n+++ b/python/kserve/kserve/model_server.py\n@@ -12,6 +12,7 @@\n # See the License for the specific language governing permissions and\n # limitations under the License.\n \n+import sys\n import argparse\n import asyncio\n import concurrent.futures\n@@ -153,7 +154,12 @@\n         async def serve():\n             logger.info(f\"Starting uvicorn with {self.workers} workers\")\n             loop = asyncio.get_event_loop()\n-            for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:\n+            if sys.platform not in ['win32', 'win64']:\n+                sig_list = [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]\n+            else:\n+                sig_list = [signal.SIGINT, signal.SIGTERM]\n+\n+            for sig in sig_list:\n                 loop.add_signal_handler(\n                     sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))\n                 )\n", "issue": "Kserve ModelServer().start(list_of_models) fails on Windows\n/kind bug\r\n\r\n**What steps did you take and what happened:**\r\nWhen ModelServer().start(list_of_models) is called, exception is raised at this location: https://github.com/kserve/kserve/blob/0ba642f9b6fa40b38fe272d04c8dfde3598bfb05/python/kserve/kserve/model_server.py#L154\r\n\r\n**What did you expect to happen:**\r\n\r\nThis simple workaround makes it possible to run the code on Windows:\r\n```\r\n            # SIGQUIT is not available on Windows\r\n            try:\r\n                sig_list = [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]\r\n            except:\r\n                sig_list = [signal.SIGINT, signal.SIGTERM]\r\n\r\n            # add_signal_handler may not have been implemented\r\n            try:\r\n                for sig in sig_list:\r\n                    loop.add_signal_handler(\r\n                        sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))\r\n                    )\r\n            except:\r\n                pass\r\n```\r\n\r\nEven though Windows-based containers are very unlikely to be used for deploying KServe inference services, the ability to run kserve-based code locally on Windows-based machines for development/debugging purposes often comes very handy. I suggest fixing the Windows incompatibility issue in the next release of the kserve Python package.\r\n\r\n**Environment:**\r\n\r\n- OS (e.g. from `/etc/os-release`): Windows 10\r\n\n", "before_files": [{"content": "# Copyright 2022 The KServe Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport argparse\nimport asyncio\nimport concurrent.futures\nimport multiprocessing\nimport signal\nimport socket\nfrom multiprocessing import Process\nfrom typing import Dict, List, Optional, Union\n\nfrom ray import serve as rayserve\nfrom ray.serve.api import Deployment\nfrom ray.serve.handle import RayServeHandle\n\nfrom .logging import KSERVE_LOG_CONFIG, logger\nfrom .model import Model\nfrom .model_repository import ModelRepository\nfrom .protocol.dataplane import DataPlane\nfrom .protocol.grpc.server import GRPCServer\nfrom .protocol.model_repository_extension import ModelRepositoryExtension\nfrom .protocol.rest.server import UvicornServer\nfrom .utils import utils\n\nDEFAULT_HTTP_PORT = 8080\nDEFAULT_GRPC_PORT = 8081\n\nparser = argparse.ArgumentParser(add_help=False)\nparser.add_argument(\"--http_port\", default=DEFAULT_HTTP_PORT, type=int,\n                    help=\"The HTTP Port listened to by the model server.\")\nparser.add_argument(\"--grpc_port\", default=DEFAULT_GRPC_PORT, type=int,\n                    help=\"The GRPC Port listened to by the model server.\")\nparser.add_argument(\"--workers\", default=1, type=int,\n                    help=\"The number of workers for multi-processing.\")\nparser.add_argument(\"--max_threads\", default=4, type=int,\n                    help=\"The number of max processing threads in each worker.\")\nparser.add_argument('--max_asyncio_workers', default=None, type=int,\n                    help='Max number of asyncio workers to spawn')\nparser.add_argument(\"--enable_grpc\", default=True, type=lambda x: utils.strtobool(x),\n                    help=\"Enable gRPC for the model server\")\nparser.add_argument(\"--enable_docs_url\", default=False, type=lambda x: utils.strtobool(x),\n                    help=\"Enable docs url '/docs' to display Swagger UI.\")\nparser.add_argument(\"--enable_latency_logging\", default=True, type=lambda x: utils.strtobool(x),\n                    help=\"Output a log per request with latency metrics.\")\nparser.add_argument(\"--configure_logging\", default=True, type=lambda x: utils.strtobool(x),\n                    help=\"Whether to configure KServe and Uvicorn logging\")\nparser.add_argument(\"--log_config_file\", default=None, type=str,\n                    help=\"File path containing UvicornServer's log config. Needs to be a yaml or json file.\")\nparser.add_argument(\"--access_log_format\", default=None, type=str,\n                    help=\"Format to set for the access log (provided by asgi-logger).\")\n\nargs, _ = parser.parse_known_args()\n\n\nclass ModelServer:\n    \"\"\"KServe ModelServer\n\n    Args:\n        http_port (int): HTTP port. Default: ``8080``.\n        grpc_port (int): GRPC port. Default: ``8081``.\n        workers (int): Number of workers for uvicorn. Default: ``1``.\n        max_threads (int): Max number of processing threads. Default: ``4``\n        max_asyncio_workers (int): Max number of AsyncIO threads. Default: ``None``\n        registered_models (ModelRepository): Model repository with registered models.\n        enable_grpc (bool): Whether to turn on grpc server. Default: ``True``\n        enable_docs_url (bool): Whether to turn on ``/docs`` Swagger UI. Default: ``False``.\n        enable_latency_logging (bool): Whether to log latency metric. Default: ``True``.\n        configure_logging (bool): Whether to configure KServe and Uvicorn logging. Default: ``True``.\n        log_config (dict or str): File path or dict containing log config. Default: ``None``.\n        access_log_format (string): Format to set for the access log (provided by asgi-logger). Default: ``None``\n    \"\"\"\n\n    def __init__(self, http_port: int = args.http_port,\n                 grpc_port: int = args.grpc_port,\n                 workers: int = args.workers,\n                 max_threads: int = args.max_threads,\n                 max_asyncio_workers: int = args.max_asyncio_workers,\n                 registered_models: ModelRepository = ModelRepository(),\n                 enable_grpc: bool = args.enable_grpc,\n                 enable_docs_url: bool = args.enable_docs_url,\n                 enable_latency_logging: bool = args.enable_latency_logging,\n                 configure_logging: bool = args.configure_logging,\n                 log_config: Optional[Union[Dict, str]] = args.log_config_file,\n                 access_log_format: str = args.access_log_format):\n        self.registered_models = registered_models\n        self.http_port = http_port\n        self.grpc_port = grpc_port\n        self.workers = workers\n        self.max_threads = max_threads\n        self.max_asyncio_workers = max_asyncio_workers\n        self.enable_grpc = enable_grpc\n        self.enable_docs_url = enable_docs_url\n        self.enable_latency_logging = enable_latency_logging\n        self.dataplane = DataPlane(model_registry=registered_models)\n        self.model_repository_extension = ModelRepositoryExtension(\n            model_registry=self.registered_models)\n        self._grpc_server = None\n        if self.enable_grpc:\n            self._grpc_server = GRPCServer(grpc_port, self.dataplane,\n                                           self.model_repository_extension)\n\n        # Logs can be passed as a path to a file or a dictConfig.\n        # We rely on Uvicorn to configure the loggers for us.\n        if configure_logging:\n            self.log_config = log_config if log_config is not None else KSERVE_LOG_CONFIG\n        else:\n            # By setting log_config to None we tell Uvicorn not to configure logging\n            self.log_config = None\n\n        self.access_log_format = access_log_format\n\n    def start(self, models: Union[List[Model], Dict[str, Deployment]]) -> None:\n        if isinstance(models, list):\n            for model in models:\n                if isinstance(model, Model):\n                    self.register_model(model)\n                    # pass whether to log request latency into the model\n                    model.enable_latency_logging = self.enable_latency_logging\n                else:\n                    raise RuntimeError(\"Model type should be 'Model'\")\n        elif isinstance(models, dict):\n            if all([isinstance(v, Deployment) for v in models.values()]):\n                # TODO: make this port number a variable\n                rayserve.start(detached=True, http_options={\"host\": \"0.0.0.0\", \"port\": 9071})\n                for key in models:\n                    models[key].deploy()\n                    handle = models[key].get_handle()\n                    self.register_model_handle(key, handle)\n            else:\n                raise RuntimeError(\"Model type should be RayServe Deployment\")\n        else:\n            raise RuntimeError(\"Unknown model collection types\")\n\n        if self.max_asyncio_workers is None:\n            # formula as suggest in https://bugs.python.org/issue35279\n            self.max_asyncio_workers = min(32, utils.cpu_count() + 4)\n        logger.info(f\"Setting max asyncio worker threads as {self.max_asyncio_workers}\")\n        asyncio.get_event_loop().set_default_executor(\n            concurrent.futures.ThreadPoolExecutor(max_workers=self.max_asyncio_workers))\n\n        async def serve():\n            logger.info(f\"Starting uvicorn with {self.workers} workers\")\n            loop = asyncio.get_event_loop()\n            for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:\n                loop.add_signal_handler(\n                    sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))\n                )\n            if self.workers == 1:\n                self._rest_server = UvicornServer(self.http_port, [],\n                                                  self.dataplane, self.model_repository_extension,\n                                                  self.enable_docs_url,\n                                                  log_config=self.log_config,\n                                                  access_log_format=self.access_log_format)\n                await self._rest_server.run()\n            else:\n                # Since py38 MacOS/Windows defaults to use spawn for starting multiprocessing.\n                # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods\n                # Spawn does not work with FastAPI/uvicorn in multiprocessing mode, use fork for multiprocessing\n                # https://github.com/tiangolo/fastapi/issues/1586\n                serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n                serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n                serversocket.bind(('0.0.0.0', self.http_port))\n                serversocket.listen(5)\n                multiprocessing.set_start_method('fork')\n                server = UvicornServer(self.http_port, [serversocket],\n                                       self.dataplane, self.model_repository_extension,\n                                       self.enable_docs_url, log_config=self.log_config,\n                                       access_log_format=self.access_log_format)\n                for _ in range(self.workers):\n                    p = Process(target=server.run_sync)\n                    p.start()\n\n        async def servers_task():\n            servers = [serve()]\n            if self.enable_grpc:\n                servers.append(self._grpc_server.start(self.max_threads))\n            await asyncio.gather(*servers)\n\n        asyncio.run(servers_task())\n\n    async def stop(self, sig: Optional[int] = None):\n        logger.info(\"Stopping the model server\")\n        if self._rest_server:\n            logger.info(\"Stopping the rest server\")\n            await self._rest_server.stop()\n        if self._grpc_server:\n            logger.info(\"Stopping the grpc server\")\n            await self._grpc_server.stop(sig)\n\n    def register_model_handle(self, name: str, model_handle: RayServeHandle):\n        self.registered_models.update_handle(name, model_handle)\n        logger.info(\"Registering model handle: %s\", name)\n\n    def register_model(self, model: Model):\n        if not model.name:\n            raise Exception(\n                \"Failed to register model, model.name must be provided.\")\n        self.registered_models.update(model)\n        logger.info(\"Registering model: %s\", model.name)\n", "path": "python/kserve/kserve/model_server.py"}], "after_files": [{"content": "# Copyright 2022 The KServe Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport argparse\nimport asyncio\nimport concurrent.futures\nimport multiprocessing\nimport signal\nimport socket\nfrom multiprocessing import Process\nfrom typing import Dict, List, Optional, Union\n\nfrom ray import serve as rayserve\nfrom ray.serve.api import Deployment\nfrom ray.serve.handle import RayServeHandle\n\nfrom .logging import KSERVE_LOG_CONFIG, logger\nfrom .model import Model\nfrom .model_repository import ModelRepository\nfrom .protocol.dataplane import DataPlane\nfrom .protocol.grpc.server import GRPCServer\nfrom .protocol.model_repository_extension import ModelRepositoryExtension\nfrom .protocol.rest.server import UvicornServer\nfrom .utils import utils\n\nDEFAULT_HTTP_PORT = 8080\nDEFAULT_GRPC_PORT = 8081\n\nparser = argparse.ArgumentParser(add_help=False)\nparser.add_argument(\"--http_port\", default=DEFAULT_HTTP_PORT, type=int,\n                    help=\"The HTTP Port listened to by the model server.\")\nparser.add_argument(\"--grpc_port\", default=DEFAULT_GRPC_PORT, type=int,\n                    help=\"The GRPC Port listened to by the model server.\")\nparser.add_argument(\"--workers\", default=1, type=int,\n                    help=\"The number of workers for multi-processing.\")\nparser.add_argument(\"--max_threads\", default=4, type=int,\n                    help=\"The number of max processing threads in each worker.\")\nparser.add_argument('--max_asyncio_workers', default=None, type=int,\n                    help='Max number of asyncio workers to spawn')\nparser.add_argument(\"--enable_grpc\", default=True, type=lambda x: utils.strtobool(x),\n                    help=\"Enable gRPC for the model server\")\nparser.add_argument(\"--enable_docs_url\", default=False, type=lambda x: utils.strtobool(x),\n                    help=\"Enable docs url '/docs' to display Swagger UI.\")\nparser.add_argument(\"--enable_latency_logging\", default=True, type=lambda x: utils.strtobool(x),\n                    help=\"Output a log per request with latency metrics.\")\nparser.add_argument(\"--configure_logging\", default=True, type=lambda x: utils.strtobool(x),\n                    help=\"Whether to configure KServe and Uvicorn logging\")\nparser.add_argument(\"--log_config_file\", default=None, type=str,\n                    help=\"File path containing UvicornServer's log config. Needs to be a yaml or json file.\")\nparser.add_argument(\"--access_log_format\", default=None, type=str,\n                    help=\"Format to set for the access log (provided by asgi-logger).\")\n\nargs, _ = parser.parse_known_args()\n\n\nclass ModelServer:\n    \"\"\"KServe ModelServer\n\n    Args:\n        http_port (int): HTTP port. Default: ``8080``.\n        grpc_port (int): GRPC port. Default: ``8081``.\n        workers (int): Number of workers for uvicorn. Default: ``1``.\n        max_threads (int): Max number of processing threads. Default: ``4``\n        max_asyncio_workers (int): Max number of AsyncIO threads. Default: ``None``\n        registered_models (ModelRepository): Model repository with registered models.\n        enable_grpc (bool): Whether to turn on grpc server. Default: ``True``\n        enable_docs_url (bool): Whether to turn on ``/docs`` Swagger UI. Default: ``False``.\n        enable_latency_logging (bool): Whether to log latency metric. Default: ``True``.\n        configure_logging (bool): Whether to configure KServe and Uvicorn logging. Default: ``True``.\n        log_config (dict or str): File path or dict containing log config. Default: ``None``.\n        access_log_format (string): Format to set for the access log (provided by asgi-logger). Default: ``None``\n    \"\"\"\n\n    def __init__(self, http_port: int = args.http_port,\n                 grpc_port: int = args.grpc_port,\n                 workers: int = args.workers,\n                 max_threads: int = args.max_threads,\n                 max_asyncio_workers: int = args.max_asyncio_workers,\n                 registered_models: ModelRepository = ModelRepository(),\n                 enable_grpc: bool = args.enable_grpc,\n                 enable_docs_url: bool = args.enable_docs_url,\n                 enable_latency_logging: bool = args.enable_latency_logging,\n                 configure_logging: bool = args.configure_logging,\n                 log_config: Optional[Union[Dict, str]] = args.log_config_file,\n                 access_log_format: str = args.access_log_format):\n        self.registered_models = registered_models\n        self.http_port = http_port\n        self.grpc_port = grpc_port\n        self.workers = workers\n        self.max_threads = max_threads\n        self.max_asyncio_workers = max_asyncio_workers\n        self.enable_grpc = enable_grpc\n        self.enable_docs_url = enable_docs_url\n        self.enable_latency_logging = enable_latency_logging\n        self.dataplane = DataPlane(model_registry=registered_models)\n        self.model_repository_extension = ModelRepositoryExtension(\n            model_registry=self.registered_models)\n        self._grpc_server = None\n        if self.enable_grpc:\n            self._grpc_server = GRPCServer(grpc_port, self.dataplane,\n                                           self.model_repository_extension)\n\n        # Logs can be passed as a path to a file or a dictConfig.\n        # We rely on Uvicorn to configure the loggers for us.\n        if configure_logging:\n            self.log_config = log_config if log_config is not None else KSERVE_LOG_CONFIG\n        else:\n            # By setting log_config to None we tell Uvicorn not to configure logging\n            self.log_config = None\n\n        self.access_log_format = access_log_format\n\n    def start(self, models: Union[List[Model], Dict[str, Deployment]]) -> None:\n        if isinstance(models, list):\n            for model in models:\n                if isinstance(model, Model):\n                    self.register_model(model)\n                    # pass whether to log request latency into the model\n                    model.enable_latency_logging = self.enable_latency_logging\n                else:\n                    raise RuntimeError(\"Model type should be 'Model'\")\n        elif isinstance(models, dict):\n            if all([isinstance(v, Deployment) for v in models.values()]):\n                # TODO: make this port number a variable\n                rayserve.start(detached=True, http_options={\"host\": \"0.0.0.0\", \"port\": 9071})\n                for key in models:\n                    models[key].deploy()\n                    handle = models[key].get_handle()\n                    self.register_model_handle(key, handle)\n            else:\n                raise RuntimeError(\"Model type should be RayServe Deployment\")\n        else:\n            raise RuntimeError(\"Unknown model collection types\")\n\n        if self.max_asyncio_workers is None:\n            # formula as suggest in https://bugs.python.org/issue35279\n            self.max_asyncio_workers = min(32, utils.cpu_count() + 4)\n        logger.info(f\"Setting max asyncio worker threads as {self.max_asyncio_workers}\")\n        asyncio.get_event_loop().set_default_executor(\n            concurrent.futures.ThreadPoolExecutor(max_workers=self.max_asyncio_workers))\n\n        async def serve():\n            logger.info(f\"Starting uvicorn with {self.workers} workers\")\n            loop = asyncio.get_event_loop()\n            if sys.platform not in ['win32', 'win64']:\n                sig_list = [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]\n            else:\n                sig_list = [signal.SIGINT, signal.SIGTERM]\n\n            for sig in sig_list:\n                loop.add_signal_handler(\n                    sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))\n                )\n            if self.workers == 1:\n                self._rest_server = UvicornServer(self.http_port, [],\n                                                  self.dataplane, self.model_repository_extension,\n                                                  self.enable_docs_url,\n                                                  log_config=self.log_config,\n                                                  access_log_format=self.access_log_format)\n                await self._rest_server.run()\n            else:\n                # Since py38 MacOS/Windows defaults to use spawn for starting multiprocessing.\n                # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods\n                # Spawn does not work with FastAPI/uvicorn in multiprocessing mode, use fork for multiprocessing\n                # https://github.com/tiangolo/fastapi/issues/1586\n                serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n                serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n                serversocket.bind(('0.0.0.0', self.http_port))\n                serversocket.listen(5)\n                multiprocessing.set_start_method('fork')\n                server = UvicornServer(self.http_port, [serversocket],\n                                       self.dataplane, self.model_repository_extension,\n                                       self.enable_docs_url, log_config=self.log_config,\n                                       access_log_format=self.access_log_format)\n                for _ in range(self.workers):\n                    p = Process(target=server.run_sync)\n                    p.start()\n\n        async def servers_task():\n            servers = [serve()]\n            if self.enable_grpc:\n                servers.append(self._grpc_server.start(self.max_threads))\n            await asyncio.gather(*servers)\n\n        asyncio.run(servers_task())\n\n    async def stop(self, sig: Optional[int] = None):\n        logger.info(\"Stopping the model server\")\n        if self._rest_server:\n            logger.info(\"Stopping the rest server\")\n            await self._rest_server.stop()\n        if self._grpc_server:\n            logger.info(\"Stopping the grpc server\")\n            await self._grpc_server.stop(sig)\n\n    def register_model_handle(self, name: str, model_handle: RayServeHandle):\n        self.registered_models.update_handle(name, model_handle)\n        logger.info(\"Registering model handle: %s\", name)\n\n    def register_model(self, model: Model):\n        if not model.name:\n            raise Exception(\n                \"Failed to register model, model.name must be provided.\")\n        self.registered_models.update(model)\n        logger.info(\"Registering model: %s\", model.name)\n", "path": "python/kserve/kserve/model_server.py"}]}