We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
kfserving==0.6.0 python package doesn't work with Python 3.9
/kind bug

**What steps did you take and what happened:**
pip3 install kfserving==0.6.0 leads to a ray[serve] install failure, probably due to:
https://github.com/ray-project/tune-sklearn/issues/169

This happens only with Python 3.9


**What did you expect to happen:**
pip3 packages installed correctly without errors


**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]


**Environment:**

- Istio Version:
- Knative Version:
- KFServing Version: 0.6.0
- Kubeflow version: 
- Kfdef:[k8s_istio/istio_dex/gcp_basic_auth/gcp_iap/aws/aws_cognito/ibm]
- Minikube version:
- Kubernetes version: (use `kubectl version`):
- OS (e.g. from `/etc/os-release`):


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `python/kfserving/setup.py`
Content:
```
1 # Copyright 2020 kubeflow.org.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #    http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import setuptools
16 
17 TESTS_REQUIRES = [
18     'pytest',
19     'pytest-tornasync',
20     'mypy'
21 ]
22 
23 with open('requirements.txt') as f:
24     REQUIRES = f.readlines()
25 
26 setuptools.setup(
27     name='kfserving',
28     version='0.6.0',
29     author="Kubeflow Authors",
30     author_email='ellisbigelow@google.com, hejinchi@cn.ibm.com, dsun20@bloomberg.net',
31     license="Apache License Version 2.0",
32     url="https://github.com/kubeflow/kfserving/tree/master/python/kfserving",
33     description="KFServing Python SDK",
34     long_description="Python SDK for KFServing Server and Client.",
35     python_requires='>=3.6',
36     packages=[
37         'kfserving',
38         'kfserving.api',
39         'kfserving.constants',
40         'kfserving.models',
41         'kfserving.handlers',
42         'kfserving.utils',
43     ],
44     package_data={'': ['requirements.txt']},
45     include_package_data=True,
46     zip_safe=False,
47     classifiers=[
48         'Intended Audience :: Developers',
49         'Intended Audience :: Education',
50         'Intended Audience :: Science/Research',
51         'Programming Language :: Python :: 3',
52         'Programming Language :: Python :: 3.6',
53         'Programming Language :: Python :: 3.7',
54         "License :: OSI Approved :: Apache Software License",
55         "Operating System :: OS Independent",
56         'Topic :: Scientific/Engineering',
57         'Topic :: Scientific/Engineering :: Artificial Intelligence',
58         'Topic :: Software Development',
59         'Topic :: Software Development :: Libraries',
60         'Topic :: Software Development :: Libraries :: Python Modules',
61     ],
62     install_requires=REQUIRES,
63     tests_require=TESTS_REQUIRES,
64     extras_require={'test': TESTS_REQUIRES}
65 )
66
```

Path: `python/kfserving/kfserving/kfserver.py`
Content:
```
1 # Copyright 2020 kubeflow.org.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #    http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import argparse
16 import logging
17 import json
18 import inspect
19 import sys
20 from typing import List, Optional, Dict, Union
21 import tornado.ioloop
22 import tornado.web
23 import tornado.httpserver
24 import tornado.log
25 import asyncio
26 from tornado import concurrent
27 from .utils import utils
28 
29 from kfserving.handlers.http import PredictHandler, ExplainHandler
30 from kfserving import KFModel
31 from kfserving.kfmodel_repository import KFModelRepository
32 from ray.serve.api import RayServeHandle, ServeDeployment
33 from ray import serve
34 
35 DEFAULT_HTTP_PORT = 8080
36 DEFAULT_GRPC_PORT = 8081
37 DEFAULT_MAX_BUFFER_SIZE = 104857600
38 
39 parser = argparse.ArgumentParser(add_help=False)
40 parser.add_argument('--http_port', default=DEFAULT_HTTP_PORT, type=int,
41                     help='The HTTP Port listened to by the model server.')
42 parser.add_argument('--grpc_port', default=DEFAULT_GRPC_PORT, type=int,
43                     help='The GRPC Port listened to by the model server.')
44 parser.add_argument('--max_buffer_size', default=DEFAULT_MAX_BUFFER_SIZE, type=int,
45                     help='The max buffer size for tornado.')
46 parser.add_argument('--workers', default=1, type=int,
47                     help='The number of works to fork')
48 parser.add_argument('--max_asyncio_workers', default=None, type=int,
49                     help='Max number of asyncio workers to spawn')
50 args, _ = parser.parse_known_args()
51 
52 tornado.log.enable_pretty_logging()
53 
54 
55 class KFServer:
56     def __init__(self, http_port: int = args.http_port,
57                  grpc_port: int = args.grpc_port,
58                  max_buffer_size: int = args.max_buffer_size,
59                  workers: int = args.workers,
60                  max_asyncio_workers: int = args.max_asyncio_workers,
61                  registered_models: KFModelRepository = KFModelRepository()):
62         self.registered_models = registered_models
63         self.http_port = http_port
64         self.grpc_port = grpc_port
65         self.max_buffer_size = max_buffer_size
66         self.workers = workers
67         self.max_asyncio_workers = max_asyncio_workers
68         self._http_server: Optional[tornado.httpserver.HTTPServer] = None
69 
70     def create_application(self):
71         return tornado.web.Application([
72             # Server Liveness API returns 200 if server is alive.
73             (r"/", LivenessHandler),
74             (r"/v2/health/live", LivenessHandler),
75             (r"/v1/models",
76              ListHandler, dict(models=self.registered_models)),
77             (r"/v2/models",
78              ListHandler, dict(models=self.registered_models)),
79             # Model Health API returns 200 if model is ready to serve.
80             (r"/v1/models/([a-zA-Z0-9_-]+)",
81              HealthHandler, dict(models=self.registered_models)),
82             (r"/v2/models/([a-zA-Z0-9_-]+)/status",
83              HealthHandler, dict(models=self.registered_models)),
84             (r"/v1/models/([a-zA-Z0-9_-]+):predict",
85              PredictHandler, dict(models=self.registered_models)),
86             (r"/v2/models/([a-zA-Z0-9_-]+)/infer",
87              PredictHandler, dict(models=self.registered_models)),
88             (r"/v1/models/([a-zA-Z0-9_-]+):explain",
89              ExplainHandler, dict(models=self.registered_models)),
90             (r"/v2/models/([a-zA-Z0-9_-]+)/explain",
91              ExplainHandler, dict(models=self.registered_models)),
92             (r"/v2/repository/models/([a-zA-Z0-9_-]+)/load",
93              LoadHandler, dict(models=self.registered_models)),
94             (r"/v2/repository/models/([a-zA-Z0-9_-]+)/unload",
95              UnloadHandler, dict(models=self.registered_models)),
96         ])
97 
98     def start(self, models: Union[List[KFModel], Dict[str, ServeDeployment]], nest_asyncio: bool = False):
99         if isinstance(models, list):
100             for model in models:
101                 if isinstance(model, KFModel):
102                     self.register_model(model)
103                 else:
104                     raise RuntimeError("Model type should be KFModel")
105         elif isinstance(models, dict):
106             if all([issubclass(v, ServeDeployment) for v in models.values()]):
107                 serve.start(detached=True, http_host='0.0.0.0', http_port=9071)
108                 for key in models:
109                     models[key].deploy()
110                     handle = models[key].get_handle()
111                     self.register_model_handle(key, handle)
112             else:
113                 raise RuntimeError("Model type should be RayServe Deployment")
114         else:
115             raise RuntimeError("Unknown model collection types")
116 
117         if self.max_asyncio_workers is None:
118             # formula as suggest in https://bugs.python.org/issue35279
119             self.max_asyncio_workers = min(32, utils.cpu_count()+4)
120 
121         logging.info(f"Setting asyncio max_workers as {self.max_asyncio_workers}")
122         asyncio.get_event_loop().set_default_executor(
123             concurrent.futures.ThreadPoolExecutor(max_workers=self.max_asyncio_workers))
124 
125         self._http_server = tornado.httpserver.HTTPServer(
126             self.create_application(), max_buffer_size=self.max_buffer_size)
127 
128         logging.info("Listening on port %s", self.http_port)
129         self._http_server.bind(self.http_port)
130         logging.info("Will fork %d workers", self.workers)
131         self._http_server.start(self.workers)
132 
133         # Need to start the IOLoop after workers have been started
134         # https://github.com/tornadoweb/tornado/issues/2426
135         # The nest_asyncio package needs to be installed by the downstream module
136         if nest_asyncio:
137             import nest_asyncio
138             nest_asyncio.apply()
139 
140         tornado.ioloop.IOLoop.current().start()
141 
142     def register_model_handle(self, name: str, model_handle: RayServeHandle):
143         self.registered_models.update_handle(name, model_handle)
144         logging.info("Registering model handle: %s", name)
145 
146     def register_model(self, model: KFModel):
147         if not model.name:
148             raise Exception(
149                 "Failed to register model, model.name must be provided.")
150         self.registered_models.update(model)
151         logging.info("Registering model: %s", model.name)
152 
153 
154 class LivenessHandler(tornado.web.RequestHandler):  # pylint:disable=too-few-public-methods
155     def get(self):
156         self.write("Alive")
157 
158 
159 class HealthHandler(tornado.web.RequestHandler):
160     def initialize(self, models: KFModelRepository):
161         self.models = models  # pylint:disable=attribute-defined-outside-init
162 
163     def get(self, name: str):
164         model = self.models.get_model(name)
165         if model is None:
166             raise tornado.web.HTTPError(
167                 status_code=404,
168                 reason="Model with name %s does not exist." % name
169             )
170 
171         if not self.models.is_model_ready(name):
172             raise tornado.web.HTTPError(
173                 status_code=503,
174                 reason="Model with name %s is not ready." % name
175             )
176 
177         self.write(json.dumps({
178             "name": model.name,
179             "ready": model.ready
180         }))
181 
182 
183 class ListHandler(tornado.web.RequestHandler):
184     def initialize(self, models: KFModelRepository):
185         self.models = models  # pylint:disable=attribute-defined-outside-init
186 
187     def get(self):
188         self.write(json.dumps([ob.name for ob in self.models.get_models()]))
189 
190 
191 class LoadHandler(tornado.web.RequestHandler):
192     def initialize(self, models: KFModelRepository):  # pylint:disable=attribute-defined-outside-init
193         self.models = models
194 
195     async def post(self, name: str):
196         try:
197             if inspect.iscoroutinefunction(self.models.load):
198                 await self.models.load(name)
199             else:
200                 self.models.load(name)
201         except Exception:
202             ex_type, ex_value, ex_traceback = sys.exc_info()
203             raise tornado.web.HTTPError(
204                 status_code=500,
205                 reason=f"Model with name {name} is not ready. "
206                        f"Error type: {ex_type} error msg: {ex_value}"
207             )
208 
209         if not self.models.is_model_ready(name):
210             raise tornado.web.HTTPError(
211                 status_code=503,
212                 reason=f"Model with name {name} is not ready."
213             )
214         self.write(json.dumps({
215             "name": name,
216             "load": True
217         }))
218 
219 
220 class UnloadHandler(tornado.web.RequestHandler):
221     def initialize(self, models: KFModelRepository):  # pylint:disable=attribute-defined-outside-init
222         self.models = models
223 
224     def post(self, name: str):
225         try:
226             self.models.unload(name)
227         except KeyError:
228             raise tornado.web.HTTPError(
229                 status_code=404,
230                 reason="Model with name %s does not exist." % name
231             )
232         self.write(json.dumps({
233             "name": name,
234             "unload": True
235         }))
236
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/python/kfserving/kfserving/kfserver.py b/python/kfserving/kfserving/kfserver.py
--- a/python/kfserving/kfserving/kfserver.py
+++ b/python/kfserving/kfserving/kfserver.py
@@ -29,7 +29,7 @@
 from kfserving.handlers.http import PredictHandler, ExplainHandler
 from kfserving import KFModel
 from kfserving.kfmodel_repository import KFModelRepository
-from ray.serve.api import RayServeHandle, ServeDeployment
+from ray.serve.api import Deployment, RayServeHandle
 from ray import serve
 
 DEFAULT_HTTP_PORT = 8080
@@ -95,7 +95,7 @@
              UnloadHandler, dict(models=self.registered_models)),
         ])
 
-    def start(self, models: Union[List[KFModel], Dict[str, ServeDeployment]], nest_asyncio: bool = False):
+    def start(self, models: Union[List[KFModel], Dict[str, Deployment]], nest_asyncio: bool = False):
         if isinstance(models, list):
             for model in models:
                 if isinstance(model, KFModel):
@@ -103,7 +103,7 @@
                 else:
                     raise RuntimeError("Model type should be KFModel")
         elif isinstance(models, dict):
-            if all([issubclass(v, ServeDeployment) for v in models.values()]):
+            if all([issubclass(v, Deployment) for v in models.values()]):
                 serve.start(detached=True, http_host='0.0.0.0', http_port=9071)
                 for key in models:
                     models[key].deploy()
diff --git a/python/kfserving/setup.py b/python/kfserving/setup.py
--- a/python/kfserving/setup.py
+++ b/python/kfserving/setup.py
@@ -25,7 +25,7 @@
 
 setuptools.setup(
     name='kfserving',
-    version='0.6.0',
+    version='0.6.1',
     author="Kubeflow Authors",
     author_email='ellisbigelow@google.com, hejinchi@cn.ibm.com, dsun20@bloomberg.net',
     license="Apache License Version 2.0",

{"golden_diff": "diff --git a/python/kfserving/kfserving/kfserver.py b/python/kfserving/kfserving/kfserver.py\n--- a/python/kfserving/kfserving/kfserver.py\n+++ b/python/kfserving/kfserving/kfserver.py\n@@ -29,7 +29,7 @@\n from kfserving.handlers.http import PredictHandler, ExplainHandler\n from kfserving import KFModel\n from kfserving.kfmodel_repository import KFModelRepository\n-from ray.serve.api import RayServeHandle, ServeDeployment\n+from ray.serve.api import Deployment, RayServeHandle\n from ray import serve\n \n DEFAULT_HTTP_PORT = 8080\n@@ -95,7 +95,7 @@\n              UnloadHandler, dict(models=self.registered_models)),\n         ])\n \n-    def start(self, models: Union[List[KFModel], Dict[str, ServeDeployment]], nest_asyncio: bool = False):\n+    def start(self, models: Union[List[KFModel], Dict[str, Deployment]], nest_asyncio: bool = False):\n         if isinstance(models, list):\n             for model in models:\n                 if isinstance(model, KFModel):\n@@ -103,7 +103,7 @@\n                 else:\n                     raise RuntimeError(\"Model type should be KFModel\")\n         elif isinstance(models, dict):\n-            if all([issubclass(v, ServeDeployment) for v in models.values()]):\n+            if all([issubclass(v, Deployment) for v in models.values()]):\n                 serve.start(detached=True, http_host='0.0.0.0', http_port=9071)\n                 for key in models:\n                     models[key].deploy()\ndiff --git a/python/kfserving/setup.py b/python/kfserving/setup.py\n--- a/python/kfserving/setup.py\n+++ b/python/kfserving/setup.py\n@@ -25,7 +25,7 @@\n \n setuptools.setup(\n     name='kfserving',\n-    version='0.6.0',\n+    version='0.6.1',\n     author=\"Kubeflow Authors\",\n     author_email='ellisbigelow@google.com, hejinchi@cn.ibm.com, dsun20@bloomberg.net',\n     license=\"Apache License Version 2.0\",\n", "issue": "kfserving==0.6.0 python package doesn't work with Python 3.9\n/kind bug\r\n\r\n**What steps did you take and what happened:**\r\npip3 install kfserving==0.6.0 leads to a ray[serve] install failure, probably due to:\r\nhttps://github.com/ray-project/tune-sklearn/issues/169\r\n\r\nThis happens only with Python 3.9\r\n\r\n\r\n**What did you expect to happen:**\r\npip3 packages installed correctly without errors\r\n\r\n\r\n**Anything else you would like to add:**\r\n[Miscellaneous information that will assist in solving the issue.]\r\n\r\n\r\n**Environment:**\r\n\r\n- Istio Version:\r\n- Knative Version:\r\n- KFServing Version: 0.6.0\r\n- Kubeflow version: \r\n- Kfdef:[k8s_istio/istio_dex/gcp_basic_auth/gcp_iap/aws/aws_cognito/ibm]\r\n- Minikube version:\r\n- Kubernetes version: (use `kubectl version`):\r\n- OS (e.g. from `/etc/os-release`):\r\n\n", "before_files": [{"content": "# Copyright 2020 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport setuptools\n\nTESTS_REQUIRES = [\n    'pytest',\n    'pytest-tornasync',\n    'mypy'\n]\n\nwith open('requirements.txt') as f:\n    REQUIRES = f.readlines()\n\nsetuptools.setup(\n    name='kfserving',\n    version='0.6.0',\n    author=\"Kubeflow Authors\",\n    author_email='ellisbigelow@google.com, hejinchi@cn.ibm.com, dsun20@bloomberg.net',\n    license=\"Apache License Version 2.0\",\n    url=\"https://github.com/kubeflow/kfserving/tree/master/python/kfserving\",\n    description=\"KFServing Python SDK\",\n    long_description=\"Python SDK for KFServing Server and Client.\",\n    python_requires='>=3.6',\n    packages=[\n        'kfserving',\n        'kfserving.api',\n        'kfserving.constants',\n        'kfserving.models',\n        'kfserving.handlers',\n        'kfserving.utils',\n    ],\n    package_data={'': ['requirements.txt']},\n    include_package_data=True,\n    zip_safe=False,\n    classifiers=[\n        'Intended Audience :: Developers',\n        'Intended Audience :: Education',\n        'Intended Audience :: Science/Research',\n        'Programming Language :: Python :: 3',\n        'Programming Language :: Python :: 3.6',\n        'Programming Language :: Python :: 3.7',\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Operating System :: OS Independent\",\n        'Topic :: Scientific/Engineering',\n        'Topic :: Scientific/Engineering :: Artificial Intelligence',\n        'Topic :: Software Development',\n        'Topic :: Software Development :: Libraries',\n        'Topic :: Software Development :: Libraries :: Python Modules',\n    ],\n    install_requires=REQUIRES,\n    tests_require=TESTS_REQUIRES,\n    extras_require={'test': TESTS_REQUIRES}\n)\n", "path": "python/kfserving/setup.py"}, {"content": "# Copyright 2020 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport argparse\nimport logging\nimport json\nimport inspect\nimport sys\nfrom typing import List, Optional, Dict, Union\nimport tornado.ioloop\nimport tornado.web\nimport tornado.httpserver\nimport tornado.log\nimport asyncio\nfrom tornado import concurrent\nfrom .utils import utils\n\nfrom kfserving.handlers.http import PredictHandler, ExplainHandler\nfrom kfserving import KFModel\nfrom kfserving.kfmodel_repository import KFModelRepository\nfrom ray.serve.api import RayServeHandle, ServeDeployment\nfrom ray import serve\n\nDEFAULT_HTTP_PORT = 8080\nDEFAULT_GRPC_PORT = 8081\nDEFAULT_MAX_BUFFER_SIZE = 104857600\n\nparser = argparse.ArgumentParser(add_help=False)\nparser.add_argument('--http_port', default=DEFAULT_HTTP_PORT, type=int,\n                    help='The HTTP Port listened to by the model server.')\nparser.add_argument('--grpc_port', default=DEFAULT_GRPC_PORT, type=int,\n                    help='The GRPC Port listened to by the model server.')\nparser.add_argument('--max_buffer_size', default=DEFAULT_MAX_BUFFER_SIZE, type=int,\n                    help='The max buffer size for tornado.')\nparser.add_argument('--workers', default=1, type=int,\n                    help='The number of works to fork')\nparser.add_argument('--max_asyncio_workers', default=None, type=int,\n                    help='Max number of asyncio workers to spawn')\nargs, _ = parser.parse_known_args()\n\ntornado.log.enable_pretty_logging()\n\n\nclass KFServer:\n    def __init__(self, http_port: int = args.http_port,\n                 grpc_port: int = args.grpc_port,\n                 max_buffer_size: int = args.max_buffer_size,\n                 workers: int = args.workers,\n                 max_asyncio_workers: int = args.max_asyncio_workers,\n                 registered_models: KFModelRepository = KFModelRepository()):\n        self.registered_models = registered_models\n        self.http_port = http_port\n        self.grpc_port = grpc_port\n        self.max_buffer_size = max_buffer_size\n        self.workers = workers\n        self.max_asyncio_workers = max_asyncio_workers\n        self._http_server: Optional[tornado.httpserver.HTTPServer] = None\n\n    def create_application(self):\n        return tornado.web.Application([\n            # Server Liveness API returns 200 if server is alive.\n            (r\"/\", LivenessHandler),\n            (r\"/v2/health/live\", LivenessHandler),\n            (r\"/v1/models\",\n             ListHandler, dict(models=self.registered_models)),\n            (r\"/v2/models\",\n             ListHandler, dict(models=self.registered_models)),\n            # Model Health API returns 200 if model is ready to serve.\n            (r\"/v1/models/([a-zA-Z0-9_-]+)\",\n             HealthHandler, dict(models=self.registered_models)),\n            (r\"/v2/models/([a-zA-Z0-9_-]+)/status\",\n             HealthHandler, dict(models=self.registered_models)),\n            (r\"/v1/models/([a-zA-Z0-9_-]+):predict\",\n             PredictHandler, dict(models=self.registered_models)),\n            (r\"/v2/models/([a-zA-Z0-9_-]+)/infer\",\n             PredictHandler, dict(models=self.registered_models)),\n            (r\"/v1/models/([a-zA-Z0-9_-]+):explain\",\n             ExplainHandler, dict(models=self.registered_models)),\n            (r\"/v2/models/([a-zA-Z0-9_-]+)/explain\",\n             ExplainHandler, dict(models=self.registered_models)),\n            (r\"/v2/repository/models/([a-zA-Z0-9_-]+)/load\",\n             LoadHandler, dict(models=self.registered_models)),\n            (r\"/v2/repository/models/([a-zA-Z0-9_-]+)/unload\",\n             UnloadHandler, dict(models=self.registered_models)),\n        ])\n\n    def start(self, models: Union[List[KFModel], Dict[str, ServeDeployment]], nest_asyncio: bool = False):\n        if isinstance(models, list):\n            for model in models:\n                if isinstance(model, KFModel):\n                    self.register_model(model)\n                else:\n                    raise RuntimeError(\"Model type should be KFModel\")\n        elif isinstance(models, dict):\n            if all([issubclass(v, ServeDeployment) for v in models.values()]):\n                serve.start(detached=True, http_host='0.0.0.0', http_port=9071)\n                for key in models:\n                    models[key].deploy()\n                    handle = models[key].get_handle()\n                    self.register_model_handle(key, handle)\n            else:\n                raise RuntimeError(\"Model type should be RayServe Deployment\")\n        else:\n            raise RuntimeError(\"Unknown model collection types\")\n\n        if self.max_asyncio_workers is None:\n            # formula as suggest in https://bugs.python.org/issue35279\n            self.max_asyncio_workers = min(32, utils.cpu_count()+4)\n\n        logging.info(f\"Setting asyncio max_workers as {self.max_asyncio_workers}\")\n        asyncio.get_event_loop().set_default_executor(\n            concurrent.futures.ThreadPoolExecutor(max_workers=self.max_asyncio_workers))\n\n        self._http_server = tornado.httpserver.HTTPServer(\n            self.create_application(), max_buffer_size=self.max_buffer_size)\n\n        logging.info(\"Listening on port %s\", self.http_port)\n        self._http_server.bind(self.http_port)\n        logging.info(\"Will fork %d workers\", self.workers)\n        self._http_server.start(self.workers)\n\n        # Need to start the IOLoop after workers have been started\n        # https://github.com/tornadoweb/tornado/issues/2426\n        # The nest_asyncio package needs to be installed by the downstream module\n        if nest_asyncio:\n            import nest_asyncio\n            nest_asyncio.apply()\n\n        tornado.ioloop.IOLoop.current().start()\n\n    def register_model_handle(self, name: str, model_handle: RayServeHandle):\n        self.registered_models.update_handle(name, model_handle)\n        logging.info(\"Registering model handle: %s\", name)\n\n    def register_model(self, model: KFModel):\n        if not model.name:\n            raise Exception(\n                \"Failed to register model, model.name must be provided.\")\n        self.registered_models.update(model)\n        logging.info(\"Registering model: %s\", model.name)\n\n\nclass LivenessHandler(tornado.web.RequestHandler):  # pylint:disable=too-few-public-methods\n    def get(self):\n        self.write(\"Alive\")\n\n\nclass HealthHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):\n        self.models = models  # pylint:disable=attribute-defined-outside-init\n\n    def get(self, name: str):\n        model = self.models.get_model(name)\n        if model is None:\n            raise tornado.web.HTTPError(\n                status_code=404,\n                reason=\"Model with name %s does not exist.\" % name\n            )\n\n        if not self.models.is_model_ready(name):\n            raise tornado.web.HTTPError(\n                status_code=503,\n                reason=\"Model with name %s is not ready.\" % name\n            )\n\n        self.write(json.dumps({\n            \"name\": model.name,\n            \"ready\": model.ready\n        }))\n\n\nclass ListHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):\n        self.models = models  # pylint:disable=attribute-defined-outside-init\n\n    def get(self):\n        self.write(json.dumps([ob.name for ob in self.models.get_models()]))\n\n\nclass LoadHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):  # pylint:disable=attribute-defined-outside-init\n        self.models = models\n\n    async def post(self, name: str):\n        try:\n            if inspect.iscoroutinefunction(self.models.load):\n                await self.models.load(name)\n            else:\n                self.models.load(name)\n        except Exception:\n            ex_type, ex_value, ex_traceback = sys.exc_info()\n            raise tornado.web.HTTPError(\n                status_code=500,\n                reason=f\"Model with name {name} is not ready. \"\n                       f\"Error type: {ex_type} error msg: {ex_value}\"\n            )\n\n        if not self.models.is_model_ready(name):\n            raise tornado.web.HTTPError(\n                status_code=503,\n                reason=f\"Model with name {name} is not ready.\"\n            )\n        self.write(json.dumps({\n            \"name\": name,\n            \"load\": True\n        }))\n\n\nclass UnloadHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):  # pylint:disable=attribute-defined-outside-init\n        self.models = models\n\n    def post(self, name: str):\n        try:\n            self.models.unload(name)\n        except KeyError:\n            raise tornado.web.HTTPError(\n                status_code=404,\n                reason=\"Model with name %s does not exist.\" % name\n            )\n        self.write(json.dumps({\n            \"name\": name,\n            \"unload\": True\n        }))\n", "path": "python/kfserving/kfserving/kfserver.py"}], "after_files": [{"content": "# Copyright 2020 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport setuptools\n\nTESTS_REQUIRES = [\n    'pytest',\n    'pytest-tornasync',\n    'mypy'\n]\n\nwith open('requirements.txt') as f:\n    REQUIRES = f.readlines()\n\nsetuptools.setup(\n    name='kfserving',\n    version='0.6.1',\n    author=\"Kubeflow Authors\",\n    author_email='ellisbigelow@google.com, hejinchi@cn.ibm.com, dsun20@bloomberg.net',\n    license=\"Apache License Version 2.0\",\n    url=\"https://github.com/kubeflow/kfserving/tree/master/python/kfserving\",\n    description=\"KFServing Python SDK\",\n    long_description=\"Python SDK for KFServing Server and Client.\",\n    python_requires='>=3.6',\n    packages=[\n        'kfserving',\n        'kfserving.api',\n        'kfserving.constants',\n        'kfserving.models',\n        'kfserving.handlers',\n        'kfserving.utils',\n    ],\n    package_data={'': ['requirements.txt']},\n    include_package_data=True,\n    zip_safe=False,\n    classifiers=[\n        'Intended Audience :: Developers',\n        'Intended Audience :: Education',\n        'Intended Audience :: Science/Research',\n        'Programming Language :: Python :: 3',\n        'Programming Language :: Python :: 3.6',\n        'Programming Language :: Python :: 3.7',\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Operating System :: OS Independent\",\n        'Topic :: Scientific/Engineering',\n        'Topic :: Scientific/Engineering :: Artificial Intelligence',\n        'Topic :: Software Development',\n        'Topic :: Software Development :: Libraries',\n        'Topic :: Software Development :: Libraries :: Python Modules',\n    ],\n    install_requires=REQUIRES,\n    tests_require=TESTS_REQUIRES,\n    extras_require={'test': TESTS_REQUIRES}\n)\n", "path": "python/kfserving/setup.py"}, {"content": "# Copyright 2020 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport argparse\nimport logging\nimport json\nimport inspect\nimport sys\nfrom typing import List, Optional, Dict, Union\nimport tornado.ioloop\nimport tornado.web\nimport tornado.httpserver\nimport tornado.log\nimport asyncio\nfrom tornado import concurrent\nfrom .utils import utils\n\nfrom kfserving.handlers.http import PredictHandler, ExplainHandler\nfrom kfserving import KFModel\nfrom kfserving.kfmodel_repository import KFModelRepository\nfrom ray.serve.api import Deployment, RayServeHandle\nfrom ray import serve\n\nDEFAULT_HTTP_PORT = 8080\nDEFAULT_GRPC_PORT = 8081\nDEFAULT_MAX_BUFFER_SIZE = 104857600\n\nparser = argparse.ArgumentParser(add_help=False)\nparser.add_argument('--http_port', default=DEFAULT_HTTP_PORT, type=int,\n                    help='The HTTP Port listened to by the model server.')\nparser.add_argument('--grpc_port', default=DEFAULT_GRPC_PORT, type=int,\n                    help='The GRPC Port listened to by the model server.')\nparser.add_argument('--max_buffer_size', default=DEFAULT_MAX_BUFFER_SIZE, type=int,\n                    help='The max buffer size for tornado.')\nparser.add_argument('--workers', default=1, type=int,\n                    help='The number of works to fork')\nparser.add_argument('--max_asyncio_workers', default=None, type=int,\n                    help='Max number of asyncio workers to spawn')\nargs, _ = parser.parse_known_args()\n\ntornado.log.enable_pretty_logging()\n\n\nclass KFServer:\n    def __init__(self, http_port: int = args.http_port,\n                 grpc_port: int = args.grpc_port,\n                 max_buffer_size: int = args.max_buffer_size,\n                 workers: int = args.workers,\n                 max_asyncio_workers: int = args.max_asyncio_workers,\n                 registered_models: KFModelRepository = KFModelRepository()):\n        self.registered_models = registered_models\n        self.http_port = http_port\n        self.grpc_port = grpc_port\n        self.max_buffer_size = max_buffer_size\n        self.workers = workers\n        self.max_asyncio_workers = max_asyncio_workers\n        self._http_server: Optional[tornado.httpserver.HTTPServer] = None\n\n    def create_application(self):\n        return tornado.web.Application([\n            # Server Liveness API returns 200 if server is alive.\n            (r\"/\", LivenessHandler),\n            (r\"/v2/health/live\", LivenessHandler),\n            (r\"/v1/models\",\n             ListHandler, dict(models=self.registered_models)),\n            (r\"/v2/models\",\n             ListHandler, dict(models=self.registered_models)),\n            # Model Health API returns 200 if model is ready to serve.\n            (r\"/v1/models/([a-zA-Z0-9_-]+)\",\n             HealthHandler, dict(models=self.registered_models)),\n            (r\"/v2/models/([a-zA-Z0-9_-]+)/status\",\n             HealthHandler, dict(models=self.registered_models)),\n            (r\"/v1/models/([a-zA-Z0-9_-]+):predict\",\n             PredictHandler, dict(models=self.registered_models)),\n            (r\"/v2/models/([a-zA-Z0-9_-]+)/infer\",\n             PredictHandler, dict(models=self.registered_models)),\n            (r\"/v1/models/([a-zA-Z0-9_-]+):explain\",\n             ExplainHandler, dict(models=self.registered_models)),\n            (r\"/v2/models/([a-zA-Z0-9_-]+)/explain\",\n             ExplainHandler, dict(models=self.registered_models)),\n            (r\"/v2/repository/models/([a-zA-Z0-9_-]+)/load\",\n             LoadHandler, dict(models=self.registered_models)),\n            (r\"/v2/repository/models/([a-zA-Z0-9_-]+)/unload\",\n             UnloadHandler, dict(models=self.registered_models)),\n        ])\n\n    def start(self, models: Union[List[KFModel], Dict[str, Deployment]], nest_asyncio: bool = False):\n        if isinstance(models, list):\n            for model in models:\n                if isinstance(model, KFModel):\n                    self.register_model(model)\n                else:\n                    raise RuntimeError(\"Model type should be KFModel\")\n        elif isinstance(models, dict):\n            if all([issubclass(v, Deployment) for v in models.values()]):\n                serve.start(detached=True, http_host='0.0.0.0', http_port=9071)\n                for key in models:\n                    models[key].deploy()\n                    handle = models[key].get_handle()\n                    self.register_model_handle(key, handle)\n            else:\n                raise RuntimeError(\"Model type should be RayServe Deployment\")\n        else:\n            raise RuntimeError(\"Unknown model collection types\")\n\n        if self.max_asyncio_workers is None:\n            # formula as suggest in https://bugs.python.org/issue35279\n            self.max_asyncio_workers = min(32, utils.cpu_count()+4)\n\n        logging.info(f\"Setting asyncio max_workers as {self.max_asyncio_workers}\")\n        asyncio.get_event_loop().set_default_executor(\n            concurrent.futures.ThreadPoolExecutor(max_workers=self.max_asyncio_workers))\n\n        self._http_server = tornado.httpserver.HTTPServer(\n            self.create_application(), max_buffer_size=self.max_buffer_size)\n\n        logging.info(\"Listening on port %s\", self.http_port)\n        self._http_server.bind(self.http_port)\n        logging.info(\"Will fork %d workers\", self.workers)\n        self._http_server.start(self.workers)\n\n        # Need to start the IOLoop after workers have been started\n        # https://github.com/tornadoweb/tornado/issues/2426\n        # The nest_asyncio package needs to be installed by the downstream module\n        if nest_asyncio:\n            import nest_asyncio\n            nest_asyncio.apply()\n\n        tornado.ioloop.IOLoop.current().start()\n\n    def register_model_handle(self, name: str, model_handle: RayServeHandle):\n        self.registered_models.update_handle(name, model_handle)\n        logging.info(\"Registering model handle: %s\", name)\n\n    def register_model(self, model: KFModel):\n        if not model.name:\n            raise Exception(\n                \"Failed to register model, model.name must be provided.\")\n        self.registered_models.update(model)\n        logging.info(\"Registering model: %s\", model.name)\n\n\nclass LivenessHandler(tornado.web.RequestHandler):  # pylint:disable=too-few-public-methods\n    def get(self):\n        self.write(\"Alive\")\n\n\nclass HealthHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):\n        self.models = models  # pylint:disable=attribute-defined-outside-init\n\n    def get(self, name: str):\n        model = self.models.get_model(name)\n        if model is None:\n            raise tornado.web.HTTPError(\n                status_code=404,\n                reason=\"Model with name %s does not exist.\" % name\n            )\n\n        if not self.models.is_model_ready(name):\n            raise tornado.web.HTTPError(\n                status_code=503,\n                reason=\"Model with name %s is not ready.\" % name\n            )\n\n        self.write(json.dumps({\n            \"name\": model.name,\n            \"ready\": model.ready\n        }))\n\n\nclass ListHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):\n        self.models = models  # pylint:disable=attribute-defined-outside-init\n\n    def get(self):\n        self.write(json.dumps([ob.name for ob in self.models.get_models()]))\n\n\nclass LoadHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):  # pylint:disable=attribute-defined-outside-init\n        self.models = models\n\n    async def post(self, name: str):\n        try:\n            if inspect.iscoroutinefunction(self.models.load):\n                await self.models.load(name)\n            else:\n                self.models.load(name)\n        except Exception:\n            ex_type, ex_value, ex_traceback = sys.exc_info()\n            raise tornado.web.HTTPError(\n                status_code=500,\n                reason=f\"Model with name {name} is not ready. \"\n                       f\"Error type: {ex_type} error msg: {ex_value}\"\n            )\n\n        if not self.models.is_model_ready(name):\n            raise tornado.web.HTTPError(\n                status_code=503,\n                reason=f\"Model with name {name} is not ready.\"\n            )\n        self.write(json.dumps({\n            \"name\": name,\n            \"load\": True\n        }))\n\n\nclass UnloadHandler(tornado.web.RequestHandler):\n    def initialize(self, models: KFModelRepository):  # pylint:disable=attribute-defined-outside-init\n        self.models = models\n\n    def post(self, name: str):\n        try:\n            self.models.unload(name)\n        except KeyError:\n            raise tornado.web.HTTPError(\n                status_code=404,\n                reason=\"Model with name %s does not exist.\" % name\n            )\n        self.write(json.dumps({\n            \"name\": name,\n            \"unload\": True\n        }))\n", "path": "python/kfserving/kfserving/kfserver.py"}]}