Update healthcheck to use exec instead of using http healthz (#648)

* Update healthcheck to use exec instead of using http healthz

* Lint

(cherry picked from commit f4e84b7)

Author: vishalbollu

pkg/operator/workloads/api_workload.go

@@ -391,11 +391,8 @@ func tfAPISpec(
 							SuccessThreshold:    1,
 							FailureThreshold:    2,
 							Handler: kcore.Handler{
-								HTTPGet: &kcore.HTTPGetAction{
-									Path: "/healthz",
-									Port: intstr.IntOrString{
-										IntVal: defaultPortInt32,
-									},
+								Exec: &kcore.ExecAction{
+									Command: []string{"/bin/bash", "-c", "/bin/ps aux | grep \"api.py\" && test -f /health_check.txt"},
 								},
 							},
 						},
@@ -584,11 +581,8 @@ func predictorAPISpec(
 							SuccessThreshold:    1,
 							FailureThreshold:    2,
 							Handler: kcore.Handler{
-								HTTPGet: &kcore.HTTPGetAction{
-									Path: "/healthz",
-									Port: intstr.IntOrString{
-										IntVal: defaultPortInt32,
-									},
+								Exec: &kcore.ExecAction{
+									Command: []string{"/bin/bash", "-c", "/bin/ps aux | grep \"api.py\" && test -f /health_check.txt"},
 								},
 							},
 						},
@@ -745,11 +739,8 @@ func onnxAPISpec(
 							SuccessThreshold:    1,
 							FailureThreshold:    2,
 							Handler: kcore.Handler{
-								HTTPGet: &kcore.HTTPGetAction{
-									Path: "/healthz",
-									Port: intstr.IntOrString{
-										IntVal: defaultPortInt32,
-									},
+								Exec: &kcore.ExecAction{
+									Command: []string{"/bin/bash", "-c", "/bin/ps aux | grep \"api.py\" && test -f /health_check.txt"},
 								},
 							},
 						},

pkg/workloads/cortex/onnx_serve/api.py

@@ -320,6 +320,7 @@ def start(args):
     waitress_kwargs["listen"] = "*:{}".format(args.port)
 
     cx_logger().info("{} api is live".format(api["name"]))
+    open("/health_check.txt", "a").close()
     serve(app, **waitress_kwargs)
 
 

pkg/workloads/cortex/predictor_serve/api.py

@@ -161,6 +161,7 @@ def start(args):
     waitress_kwargs["listen"] = "*:{}".format(args.port)
 
     cx_logger().info("{} api is live".format(api["name"]))
+    open("/health_check.txt", "a").close()
     serve(app, **waitress_kwargs)
 
 

pkg/workloads/cortex/tf_api/api.py

@@ -457,6 +457,7 @@ def start(args):
     waitress_kwargs["listen"] = "*:{}".format(args.port)
 
     cx_logger().info("{} api is live".format(api["name"]))
+    open("/health_check.txt", "a").close()
     serve(app, **waitress_kwargs)