Merge pull request #100 from arangodb/ready-probe

Added LivenessProbe & Readiness probe

Author: ewoutp

main.go

@@ -48,6 +48,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/operator"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/arangodb/kube-arangodb/pkg/util/probe"
 	"github.com/arangodb/kube-arangodb/pkg/util/retry"
 )
 
@@ -80,6 +81,8 @@ var (
 		enableStorage    bool // Run deployment operator
 		createCRD        bool
 	}
+	deploymentProbe probe.Probe
+	storageProbe    probe.Probe
 )
 
 func init() {
@@ -135,7 +138,9 @@ func cmdMainRun(cmd *cobra.Command, args []string) {
 		cliLog.Fatal().Err(err).Msg("Failed to get hostname")
 	}
 
-	//http.HandleFunc(probe.HTTPReadyzEndpoint, probe.ReadyzHandler)
+	http.HandleFunc("/health", probe.LivenessHandler)
+	http.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler)
+	http.HandleFunc("/ready/storage", storageProbe.ReadyHandler)
 	http.Handle("/metrics", prometheus.Handler())
 	listenAddr := net.JoinHostPort(server.host, strconv.Itoa(server.port))
 	go http.ListenAndServe(listenAddr, nil)
@@ -186,11 +191,13 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 		CreateCRD:        operatorOptions.createCRD,
 	}
 	deps := operator.Dependencies{
-		LogService:    logService,
-		KubeCli:       kubecli,
-		KubeExtCli:    kubeExtCli,
-		CRCli:         crCli,
-		EventRecorder: eventRecorder,
+		LogService:      logService,
+		KubeCli:         kubecli,
+		KubeExtCli:      kubeExtCli,
+		CRCli:           crCli,
+		EventRecorder:   eventRecorder,
+		DeploymentProbe: &deploymentProbe,
+		StorageProbe:    &storageProbe,
 	}
 
 	return cfg, deps, nil

manifests/templates/deployment/deployment.yaml

@@ -6,6 +6,8 @@ metadata:
   namespace: {{ .Deployment.Operator.Namespace }}
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   template:
     metadata:
       labels:
@@ -26,3 +28,18 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        ports:
+        - name: metrics
+          containerPort: 8528
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8528
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ready/deployment
+            port: 8528
+          initialDelaySeconds: 5
+          periodSeconds: 10

manifests/templates/storage/deployment.yaml

@@ -14,6 +14,8 @@ metadata:
   namespace: {{ .Storage.Operator.Namespace }}
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   template:
     metadata:
       labels:
@@ -35,3 +37,19 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        ports:
+        - name: metrics
+          containerPort: 8528
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8528
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ready/storage
+            port: 8528
+          initialDelaySeconds: 5
+          periodSeconds: 10
+      

pkg/operator/operator.go

@@ -39,6 +39,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
 	"github.com/arangodb/kube-arangodb/pkg/logging"
 	"github.com/arangodb/kube-arangodb/pkg/storage"
+	"github.com/arangodb/kube-arangodb/pkg/util/probe"
 )
 
 const (
@@ -71,11 +72,13 @@ type Config struct {
 }
 
 type Dependencies struct {
-	LogService    logging.Service
-	KubeCli       kubernetes.Interface
-	KubeExtCli    apiextensionsclient.Interface
-	CRCli         versioned.Interface
-	EventRecorder record.EventRecorder
+	LogService      logging.Service
+	KubeCli         kubernetes.Interface
+	KubeExtCli      apiextensionsclient.Interface
+	CRCli           versioned.Interface
+	EventRecorder   record.EventRecorder
+	DeploymentProbe *probe.Probe
+	StorageProbe    *probe.Probe
 }
 
 // NewOperator instantiates a new operator from given config & dependencies.

pkg/operator/operator_deployment.go

@@ -58,6 +58,7 @@ func (o *Operator) runDeployments(stop <-chan struct{}) {
 		DeleteFunc: o.onDeleteArangoDeployment,
 	}, cache.Indexers{})
 
+	o.Dependencies.DeploymentProbe.SetReady()
 	informer.Run(stop)
 }
 

pkg/operator/operator_local_storage.go

@@ -58,6 +58,7 @@ func (o *Operator) runLocalStorages(stop <-chan struct{}) {
 		DeleteFunc: o.onDeleteArangoLocalStorage,
 	}, cache.Indexers{})
 
+	o.Dependencies.StorageProbe.SetReady()
 	informer.Run(stop)
 }
 

pkg/util/probe/health.go

@@ -0,0 +1,32 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package probe
+
+import (
+	"net/http"
+)
+
+// LivenessHandler writes back the HTTP status code 200 to indicate a healthy operator.
+func LivenessHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}

pkg/util/probe/ready.go

@@ -0,0 +1,48 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package probe
+
+import (
+	"net/http"
+	"sync/atomic"
+)
+
+// Probe wraps a readiness probe handler.
+type Probe struct {
+	ready int32
+}
+
+// SetReady marks the probe as ready.
+func (p *Probe) SetReady() {
+	atomic.StoreInt32(&p.ready, 1)
+}
+
+// ReadyHandler writes back the HTTP status code 200 if the operator is ready, and 500 otherwise.
+func (p *Probe) ReadyHandler(w http.ResponseWriter, r *http.Request) {
+	isReady := atomic.LoadInt32(&p.ready) != 0
+	if isReady {
+		w.WriteHeader(http.StatusOK)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}