[Feature] Add Startup Probe support (#836)

Author: ajanikow

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Add Graceful shutdown as finalizer (supports kubectl delete)
 - Add Watch to Lifecycle command
 - Add Topology Discovery
+- Add Support for StartupProbe
 
 ## [1.2.4](https://github.com/arangodb/kube-arangodb/tree/1.2.4) (2021-10-22)
 - Replace `beta.kubernetes.io/arch` Pod label with `kubernetes.io/arch` using Silent Rotation

pkg/apis/deployment/v1/server_group_spec.go

@@ -261,6 +261,11 @@ type ServerGroupProbesSpec struct {
 	ReadinessProbeDisabled *bool `json:"readinessProbeDisabled,omitempty"`
 	// ReadinessProbeSpec override readiness probe configuration
 	ReadinessProbeSpec *ServerGroupProbeSpec `json:"readinessProbeSpec,omitempty"`
+
+	// StartupProbeDisabled if true startupProbes are disabled
+	StartupProbeDisabled *bool `json:"startupProbeDisabled,omitempty"`
+	// StartupProbeSpec override startup probe configuration
+	StartupProbeSpec *ServerGroupProbeSpec `json:"startupProbeSpec,omitempty"`
 }
 
 // GetReadinessProbeDisabled returns in proper manner readiness probe flag with backward compatibility.

pkg/deployment/images.go

@@ -405,8 +405,8 @@ func (a *ContainerIdentity) GetPorts() []core.ContainerPort {
 	}
 }
 
-func (a *ContainerIdentity) GetProbes() (*core.Probe, *core.Probe, error) {
-	return nil, nil, nil
+func (a *ContainerIdentity) GetProbes() (*core.Probe, *core.Probe, *core.Probe, error) {
+	return nil, nil, nil, nil
 }
 
 func (a *ContainerIdentity) GetResourceRequirements() core.ResourceRequirements {

pkg/deployment/pod/probes.go

@@ -39,37 +39,47 @@ func LivenessSpec(group api.ServerGroup) Probe {
 	return probeMap[group].liveness
 }
 
+func StartupSpec(group api.ServerGroup) Probe {
+	return probeMap[group].startup
+}
+
 type Probe struct {
 	CanBeEnabled, EnabledByDefault bool
 }
 
 type probes struct {
-	liveness, readiness Probe
+	liveness, readiness, startup Probe
 }
 
 // probeMap defines default values and if Probe can be enabled
 var probeMap = map[api.ServerGroup]probes{
 	api.ServerGroupSingle: {
+		startup:   newProbe(true, false),
 		liveness:  newProbe(true, true),
 		readiness: newProbe(true, true),
 	},
 	api.ServerGroupAgents: {
+		startup:   newProbe(true, false),
 		liveness:  newProbe(true, true),
 		readiness: newProbe(true, false),
 	},
 	api.ServerGroupDBServers: {
+		startup:   newProbe(true, false),
 		liveness:  newProbe(true, true),
 		readiness: newProbe(true, false),
 	},
 	api.ServerGroupCoordinators: {
+		startup:   newProbe(true, false),
 		liveness:  newProbe(true, false),
 		readiness: newProbe(true, true),
 	},
 	api.ServerGroupSyncMasters: {
+		startup:   newProbe(true, false),
 		liveness:  newProbe(true, true),
 		readiness: newProbe(false, false),
 	},
 	api.ServerGroupSyncWorkers: {
+		startup:   newProbe(true, false),
 		liveness:  newProbe(true, true),
 		readiness: newProbe(false, false),
 	},

pkg/deployment/resources/pod_creator_arangod.go

@@ -136,17 +136,22 @@ func (a *ArangoDContainer) GetSecurityContext() *core.SecurityContext {
 	return a.groupSpec.SecurityContext.NewSecurityContext()
 }
 
-func (a *ArangoDContainer) GetProbes() (*core.Probe, *core.Probe, error) {
-	var liveness, readiness *core.Probe
+func (a *ArangoDContainer) GetProbes() (*core.Probe, *core.Probe, *core.Probe, error) {
+	var liveness, readiness, startup *core.Probe
 
 	probeLivenessConfig, err := a.resources.getLivenessProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	probeReadinessConfig, err := a.resources.getReadinessProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
+	}
+
+	probeStartupConfig, err := a.resources.getStartupProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion)
+	if err != nil {
+		return nil, nil, nil, err
 	}
 
 	if probeLivenessConfig != nil {
@@ -157,7 +162,11 @@ func (a *ArangoDContainer) GetProbes() (*core.Probe, *core.Probe, error) {
 		readiness = probeReadinessConfig.Create()
 	}
 
-	return liveness, readiness, nil
+	if probeStartupConfig != nil {
+		startup = probeStartupConfig.Create()
+	}
+
+	return liveness, readiness, startup, nil
 }
 
 func (a *ArangoDContainer) GetImage() string {
@@ -602,8 +611,8 @@ func (a *ArangoUpgradeContainer) GetName() string {
 }
 
 // GetProbes returns no probes for the ArangoD upgrade container.
-func (a *ArangoUpgradeContainer) GetProbes() (*core.Probe, *core.Probe, error) {
-	return nil, nil, nil
+func (a *ArangoUpgradeContainer) GetProbes() (*core.Probe, *core.Probe, *core.Probe, error) {
+	return nil, nil, nil, nil
 }
 
 // GetArgs returns list of arguments for the ArangoD version check container.
@@ -622,6 +631,6 @@ func (a *ArangoVersionCheckContainer) GetName() string {
 }
 
 // GetProbes returns no probes for the ArangoD version check container.
-func (a *ArangoVersionCheckContainer) GetProbes() (*core.Probe, *core.Probe, error) {
-	return nil, nil, nil
+func (a *ArangoVersionCheckContainer) GetProbes() (*core.Probe, *core.Probe, *core.Probe, error) {
+	return nil, nil, nil, nil
 }

pkg/deployment/resources/pod_creator_probes.go

@@ -49,7 +49,7 @@ type Probe interface {
 }
 
 type probeCheckBuilder struct {
-	liveness, readiness probeBuilder
+	liveness, readiness, startup probeBuilder
 }
 
 type probeBuilder func(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error)
@@ -118,6 +118,36 @@ func (r *Resources) getLivenessProbe(spec api.DeploymentSpec, group api.ServerGr
 	return config, nil
 }
 
+func (r *Resources) getStartupProbe(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
+	if !r.isStartupProbeEnabled(spec, group) {
+		return nil, nil
+	}
+
+	builders := r.probeBuilders()
+
+	builder, ok := builders[group]
+	if !ok {
+		return nil, nil
+	}
+
+	config, err := builder.startup(spec, group, version)
+	if err != nil {
+		return nil, err
+	}
+
+	groupSpec := spec.GetServerGroupSpec(group)
+
+	if !groupSpec.HasProbesSpec() {
+		return config, nil
+	}
+
+	probeSpec := groupSpec.GetProbesSpec()
+
+	config.SetSpec(probeSpec.StartupProbeSpec)
+
+	return config, nil
+}
+
 func (r *Resources) isReadinessProbeEnabled(spec api.DeploymentSpec, group api.ServerGroup) bool {
 	probe := pod.ReadinessSpec(group)
 
@@ -146,29 +176,49 @@ func (r *Resources) isLivenessProbeEnabled(spec api.DeploymentSpec, group api.Se
 	return probe.CanBeEnabled && probe.EnabledByDefault
 }
 
+func (r *Resources) isStartupProbeEnabled(spec api.DeploymentSpec, group api.ServerGroup) bool {
+	probe := pod.StartupSpec(group)
+
+	groupSpec := spec.GetServerGroupSpec(group)
+
+	if groupSpec.HasProbesSpec() {
+		if p := groupSpec.GetProbesSpec().StartupProbeDisabled; p != nil {
+			return !*p && probe.CanBeEnabled
+		}
+	}
+
+	return probe.CanBeEnabled && probe.EnabledByDefault
+}
+
 func (r *Resources) probeBuilders() map[api.ServerGroup]probeCheckBuilder {
 	return map[api.ServerGroup]probeCheckBuilder{
 		api.ServerGroupSingle: {
+			startup:   r.probeBuilderStartupCoreSelect(),
 			liveness:  r.probeBuilderLivenessCoreSelect(),
 			readiness: r.probeBuilderReadinessCoreSelect(),
 		},
 		api.ServerGroupAgents: {
+			startup:   r.probeBuilderStartupCoreSelect(),
 			liveness:  r.probeBuilderLivenessCoreSelect(),
 			readiness: r.probeBuilderReadinessSimpleCoreSelect(),
 		},
 		api.ServerGroupDBServers: {
+			startup:   r.probeBuilderStartupCoreSelect(),
 			liveness:  r.probeBuilderLivenessCoreSelect(),
 			readiness: r.probeBuilderReadinessSimpleCoreSelect(),
 		},
 		api.ServerGroupCoordinators: {
+			startup:   r.probeBuilderStartupCoreSelect(),
 			liveness:  r.probeBuilderLivenessCoreSelect(),
 			readiness: r.probeBuilderReadinessCoreSelect(),
 		},
 		api.ServerGroupSyncMasters: {
+			startup:   r.probeBuilderStartupSync,
 			liveness:  r.probeBuilderLivenessSync,
 			readiness: nilProbeBuilder,
 		},
 		api.ServerGroupSyncWorkers: {
+			startup:   r.probeBuilderStartupSync,
 			liveness:  r.probeBuilderLivenessSync,
 			readiness: nilProbeBuilder,
 		},
@@ -207,6 +257,14 @@ func (r *Resources) probeBuilderLivenessCoreSelect() probeBuilder {
 	return r.probeBuilderLivenessCore
 }
 
+func (r *Resources) probeBuilderStartupCoreSelect() probeBuilder {
+	if features.JWTRotation().Enabled() {
+		return r.probeBuilderStartupCoreOperator
+	}
+
+	return r.probeBuilderStartupCore
+}
+
 func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
 	args, err := r.probeCommand(spec, "/_api/version")
 	if err != nil {
@@ -218,6 +276,29 @@ func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, gr
 	}, nil
 }
 
+func (r *Resources) probeBuilderStartupCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
+	args, err := r.probeCommand(spec, "/_api/version")
+	if err != nil {
+		return nil, err
+	}
+
+	var retries int32
+
+	switch group {
+	case api.ServerGroupDBServers:
+		retries = 6 * 60 * 60 / 5 // Wait 6 hours for wal replay
+	default:
+		retries = 60
+	}
+
+	return &probes.CMDProbeConfig{
+		Command:             args,
+		FailureThreshold:    retries,
+		PeriodSeconds:       5,
+		InitialDelaySeconds: 1,
+	}, nil
+}
+
 func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
 	authorization := ""
 	if spec.IsAuthenticated() {
@@ -237,6 +318,38 @@ func (r *Resources) probeBuilderLivenessCore(spec api.DeploymentSpec, group api.
 	}, nil
 }
 
+func (r *Resources) probeBuilderStartupCore(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
+
+	var retries int32
+
+	switch group {
+	case api.ServerGroupDBServers:
+		retries = 6 * 60 * 60 / 5 // Wait 6 hours for wal replay
+	default:
+		retries = 60
+	}
+
+	authorization := ""
+	if spec.IsAuthenticated() {
+		secretData, err := r.getJWTSecret(spec)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		authorization, err = jwt.CreateArangodJwtAuthorizationHeaderAllowedPaths(secretData, "kube-arangodb", []string{"/_api/version"})
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+	}
+	return &probes.HTTPProbeConfig{
+		LocalPath:           "/_api/version",
+		Secure:              spec.IsSecure(),
+		Authorization:       authorization,
+		FailureThreshold:    retries,
+		PeriodSeconds:       5,
+		InitialDelaySeconds: 1,
+	}, nil
+}
+
 func (r *Resources) probeBuilderReadinessSimpleCoreSelect() probeBuilder {
 	if features.JWTRotation().Enabled() {
 		return r.probeBuilderReadinessSimpleCoreOperator
@@ -363,3 +476,38 @@ func (r *Resources) probeBuilderLivenessSync(spec api.DeploymentSpec, group api.
 		Port:          port,
 	}, nil
 }
+
+func (r *Resources) probeBuilderStartupSync(spec api.DeploymentSpec, group api.ServerGroup, version driver.Version) (Probe, error) {
+	authorization := ""
+	port := k8sutil.ArangoSyncMasterPort
+	if group == api.ServerGroupSyncWorkers {
+		port = k8sutil.ArangoSyncWorkerPort
+	}
+	if spec.Sync.Monitoring.GetTokenSecretName() != "" {
+		// Use monitoring token
+		token, err := r.getSyncMonitoringToken(spec)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		authorization = "bearer " + token
+	} else if group == api.ServerGroupSyncMasters {
+		// Fall back to JWT secret
+		secretData, err := r.getSyncJWTSecret(spec)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		authorization, err = jwt.CreateArangodJwtAuthorizationHeaderAllowedPaths(secretData, "kube-arangodb", []string{"/_api/version"})
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+	} else {
+		// Don't have a probe
+		return nil, nil
+	}
+	return &probes.HTTPProbeConfig{
+		LocalPath:     "/_api/version",
+		Secure:        spec.Sync.TLS.IsSecure(),
+		Authorization: authorization,
+		Port:          port,
+	}, nil
+}

pkg/deployment/resources/pod_creator_sync.go

@@ -105,17 +105,22 @@ func (a *ArangoSyncContainer) GetSecurityContext() *core.SecurityContext {
 	return a.groupSpec.SecurityContext.NewSecurityContext()
 }
 
-func (a *ArangoSyncContainer) GetProbes() (*core.Probe, *core.Probe, error) {
-	var liveness, readiness *core.Probe
+func (a *ArangoSyncContainer) GetProbes() (*core.Probe, *core.Probe, *core.Probe, error) {
+	var liveness, readiness, startup *core.Probe
 
 	probeLivenessConfig, err := a.resources.getLivenessProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	probeReadinessConfig, err := a.resources.getReadinessProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
+	}
+
+	probeStartupConfig, err := a.resources.getReadinessProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion)
+	if err != nil {
+		return nil, nil, nil, err
 	}
 
 	if probeLivenessConfig != nil {
@@ -126,7 +131,11 @@ func (a *ArangoSyncContainer) GetProbes() (*core.Probe, *core.Probe, error) {
 		readiness = probeReadinessConfig.Create()
 	}
 
-	return liveness, readiness, nil
+	if probeStartupConfig != nil {
+		startup = probeStartupConfig.Create()
+	}
+
+	return liveness, readiness, startup, nil
 }
 
 func (a *ArangoSyncContainer) GetResourceRequirements() core.ResourceRequirements {