Improved readiness probe, database services only use ready pods

Author: ewoutp

pkg/deployment/resources/pod_creator.go

@@ -359,7 +359,7 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve
 
 // createReadinessProbe creates configuration for a readiness probe of a server in the given group.
 func (r *Resources) createReadinessProbe(spec api.DeploymentSpec, group api.ServerGroup) (*k8sutil.HTTPProbeConfig, error) {
-	if group != api.ServerGroupCoordinators {
+	if group != api.ServerGroupSingle && group != api.ServerGroupCoordinators {
 		return nil, nil
 	}
 	authorization := ""
@@ -373,11 +373,18 @@ func (r *Resources) createReadinessProbe(spec api.DeploymentSpec, group api.Serv
 			return nil, maskAny(err)
 		}
 	}
-	return &k8sutil.HTTPProbeConfig{
-		LocalPath:     "/_api/version",
-		Secure:        spec.IsSecure(),
-		Authorization: authorization,
-	}, nil
+	probeCfg := &k8sutil.HTTPProbeConfig{
+		LocalPath:           "/_api/version",
+		Secure:              spec.IsSecure(),
+		Authorization:       authorization,
+		InitialDelaySeconds: 2,
+		PeriodSeconds:       2,
+	}
+	switch spec.GetMode() {
+	case api.DeploymentModeActiveFailover:
+		probeCfg.LocalPath = "/_admin/echo"
+	}
+	return probeCfg, nil
 }
 
 // createPodFinalizers creates a list of finalizers for a pod created for the given group.

pkg/util/k8sutil/probes.go

@@ -37,6 +37,16 @@ type HTTPProbeConfig struct {
 	Authorization string
 	// Port to inspect (defaults to ArangoPort)
 	Port int
+	// Number of seconds after the container has started before liveness probes are initiated (defaults to 30)
+	InitialDelaySeconds int32
+	// Number of seconds after which the probe times out (defaults to 2).
+	TimeoutSeconds int32
+	// How often (in seconds) to perform the probe (defaults to 10).
+	PeriodSeconds int32
+	// Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1).
+	SuccessThreshold int32
+	// Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3).
+	FailureThreshold int32
 }
 
 // Create creates a probe from given config
@@ -52,23 +62,25 @@ func (config HTTPProbeConfig) Create() *v1.Probe {
 			Value: config.Authorization,
 		})
 	}
-	port := config.Port
-	if port == 0 {
-		port = ArangoPort
+	def := func(value, defaultValue int32) int32 {
+		if value != 0 {
+			return value
+		}
+		return defaultValue
 	}
 	return &v1.Probe{
 		Handler: v1.Handler{
 			HTTPGet: &v1.HTTPGetAction{
 				Path:        config.LocalPath,
-				Port:        intstr.FromInt(port),
+				Port:        intstr.FromInt(int(def(int32(config.Port), ArangoPort))),
 				Scheme:      scheme,
 				HTTPHeaders: headers,
 			},
 		},
-		InitialDelaySeconds: 30, // Wait 30s before first probe
-		TimeoutSeconds:      2,  // Timeout of each probe is 2s
-		PeriodSeconds:       10, // Interval between probes is 10s
-		SuccessThreshold:    1,  // Single probe is enough to indicate success
-		FailureThreshold:    3,  // Need 3 failed probes to consider a failed state
+		InitialDelaySeconds: def(config.InitialDelaySeconds, 30), // Wait 30s before first probe
+		TimeoutSeconds:      def(config.TimeoutSeconds, 2),       // Timeout of each probe is 2s
+		PeriodSeconds:       def(config.PeriodSeconds, 10),       // Interval between probes is 10s
+		SuccessThreshold:    def(config.SuccessThreshold, 1),     // Single probe is enough to indicate success
+		FailureThreshold:    def(config.FailureThreshold, 3),     // Need 3 failed probes to consider a failed state
 	}
 }

pkg/util/k8sutil/services.go

@@ -23,6 +23,8 @@
 package k8sutil
 
 import (
+	"strconv"
+
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -67,7 +69,7 @@ func CreateHeadlessService(kubecli kubernetes.Interface, deployment metav1.Objec
 			Port:     ArangoPort,
 		},
 	}
-	publishNotReadyAddresses := false
+	publishNotReadyAddresses := true
 	serviceType := v1.ServiceTypeClusterIP
 	newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), ClusterIPNone, "", serviceType, ports, "", publishNotReadyAddresses, owner)
 	if err != nil {
@@ -96,8 +98,8 @@ func CreateDatabaseClientService(kubecli kubernetes.Interface, deployment metav1
 	} else {
 		role = "coordinator"
 	}
-	publishNotReadyAddresses := true
 	serviceType := v1.ServiceTypeClusterIP
+	publishNotReadyAddresses := false
 	newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, "", publishNotReadyAddresses, owner)
 	if err != nil {
 		return "", false, maskAny(err)
@@ -119,7 +121,7 @@ func CreateExternalAccessService(kubecli kubernetes.Interface, svcName, role str
 			NodePort: int32(nodePort),
 		},
 	}
-	publishNotReadyAddresses := true
+	publishNotReadyAddresses := false
 	newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, loadBalancerIP, publishNotReadyAddresses, owner)
 	if err != nil {
 		return "", false, maskAny(err)
@@ -142,7 +144,7 @@ func createService(kubecli kubernetes.Interface, svcName, deploymentName, ns, cl
 				// This annotation is deprecated, PublishNotReadyAddresses is
 				// used instead. We leave the annotation in for a while.
 				// See https://github.com/kubernetes/kubernetes/pull/49061
-				TolerateUnreadyEndpointsAnnotation: "true",
+				TolerateUnreadyEndpointsAnnotation: strconv.FormatBool(publishNotReadyAddresses),
 			},
 		},
 		Spec: v1.ServiceSpec{