Pass waitress parameters from metadata

(cherry picked from commit 5529873)

Author: deliahu

pkg/workloads/cortex/onnx_serve/api.py

@@ -308,8 +308,19 @@ def start(args):
         except Exception as e:
             cx_logger().warn("an error occurred while attempting to load classes", exc_info=True)
 
+    waitress_kwargs = {}
+    if api["tensorflow"].get("metadata") is not None:
+        for key, value in api["tensorflow"]["metadata"].items():
+            if key.startswith("waitress_"):
+                waitress_kwargs[key[len("waitress_") :]] = value
+
+    if len(waitress_kwargs) > 0:
+        cx_logger().info("waitress parameters: {}".format(waitress_kwargs))
+
+    waitress_kwargs["listen"] = "*:{}".format(args.port)
+
     cx_logger().info("{} api is live".format(api["name"]))
-    serve(app, listen="*:{}".format(args.port))
+    serve(app, **waitress_kwargs)
 
 
 def main():

pkg/workloads/cortex/predictor_serve/api.py

@@ -149,8 +149,19 @@ def start(args):
         except Exception as e:
             cx_logger().warn("an error occurred while attempting to load classes", exc_info=True)
 
+    waitress_kwargs = {}
+    if api["tensorflow"].get("metadata") is not None:
+        for key, value in api["tensorflow"]["metadata"].items():
+            if key.startswith("waitress_"):
+                waitress_kwargs[key[len("waitress_") :]] = value
+
+    if len(waitress_kwargs) > 0:
+        cx_logger().info("waitress parameters: {}".format(waitress_kwargs))
+
+    waitress_kwargs["listen"] = "*:{}".format(args.port)
+
     cx_logger().info("{} api is live".format(api["name"]))
-    serve(app, listen="*:{}".format(args.port))
+    serve(app, **waitress_kwargs)
 
 
 def main():

pkg/workloads/cortex/tf_api/api.py

@@ -445,8 +445,19 @@ def start(args):
     local_cache["parsed_signature"] = parsed_signature
     cx_logger().info("model_signature: {}".format(local_cache["parsed_signature"]))
 
+    waitress_kwargs = {}
+    if api["tensorflow"].get("metadata") is not None:
+        for key, value in api["tensorflow"]["metadata"].items():
+            if key.startswith("waitress_"):
+                waitress_kwargs[key[len("waitress_") :]] = value
+
+    if len(waitress_kwargs) > 0:
+        cx_logger().info("waitress parameters: {}".format(waitress_kwargs))
+
+    waitress_kwargs["listen"] = "*:{}".format(args.port)
+
     cx_logger().info("{} api is live".format(api["name"]))
-    serve(app, listen="*:{}".format(args.port))
+    serve(app, **waitress_kwargs)
 
 
 def main():