[Bugfix] Prevent LifeCycle restarts (#1127)

Author: ajanikow

CHANGELOG.md

@@ -5,6 +5,7 @@
 - (Feature) Add Member Update helpers
 - (Feature) Active Member condition
 - (Bugfix) Accept Initial Spec
+- (Bugfix) Prevent LifeCycle restarts
 
 ## [1.2.17](https://github.com/arangodb/kube-arangodb/tree/1.2.17) (2022-09-22)
 - (Feature) Add new field to DeploymentReplicationStatus with details on DC2DC sync status=

go.mod

@@ -55,9 +55,6 @@ require (
 	k8s.io/apimachinery v0.21.10
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog v1.0.0
-)
-
-require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
@@ -106,6 +103,10 @@ require (
 	k8s.io/utils v0.0.0-20210521133846-da695404a2bc // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.2.0
+	github.com/josephburnett/jd v1.6.1
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 )
-
-require github.com/cenkalti/backoff/v4 v4.1.3 // indirect

go.sum

@@ -164,12 +164,16 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU=
+github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
@@ -305,6 +309,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15 h1:cW/amwGEJK5MSKntPXRjX4dxs/nGxGT8gXKIsKFmHGc=
 github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15/go.mod h1:Fdm/oWRW+CH8PRbLntksCNtmcCBximKPkVQYvmMl80k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josephburnett/jd v1.6.1 h1:Uzqhcje4WqvVyp85F3Oj0ezISPTlnhnr/KaLZIy8qh0=
+github.com/josephburnett/jd v1.6.1/go.mod h1:R8ZnZnLt2D4rhW4NvBc/USTo6mzyNT6fYNIIWOJA9GY=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -339,6 +347,9 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -900,6 +911,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

pkg/deployment/resources/internal_exporter.go

@@ -19,14 +19,10 @@
 package resources
 
 import (
-	"os"
-	"path/filepath"
-
 	core "k8s.io/api/core/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
-	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
 )
@@ -36,11 +32,7 @@ func ArangodbInternalExporterContainer(image string, args []string, livenessProb
 	resources core.ResourceRequirements, securityContext *core.SecurityContext,
 	spec api.DeploymentSpec) (core.Container, error) {
 
-	binaryPath, err := os.Executable()
-	if err != nil {
-		return core.Container{}, errors.WithStack(err)
-	}
-	exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath))
+	exePath := k8sutil.LifecycleBinary()
 
 	c := core.Container{
 		Name:    shared.ExporterContainerName,

pkg/deployment/resources/pod_creator_probes.go

@@ -22,8 +22,6 @@ package resources
 
 import (
 	"math"
-	"os"
-	"path/filepath"
 	"time"
 
 	core "k8s.io/api/core/v1"
@@ -223,18 +221,13 @@ func (r *Resources) probeBuilders(imageInfo api.ImageInfo) map[api.ServerGroup]p
 	}
 }
 
-func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) ([]string, error) {
-	binaryPath, err := os.Executable()
-	if err != nil {
-		return nil, err
-	}
-	exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath))
+func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) []string {
+	exePath := k8sutil.LifecycleBinary()
 	args := []string{
 		exePath,
 		"lifecycle",
 		"probe",
 		string(probeType),
-		// TODO test rotation required when changed
 	}
 
 	if spec.IsSecure() {
@@ -245,7 +238,7 @@ func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeTyp
 		args = append(args, "--auth")
 	}
 
-	return args, nil
+	return args
 }
 
 func (r *Resources) probeBuilderLivenessCoreSelect(group api.ServerGroup, imageInfo api.ImageInfo) probeBuilder {
@@ -266,10 +259,7 @@ func (r *Resources) probeBuilderStartupCoreSelect(group api.ServerGroup, imageIn
 
 func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup,
 	image api.ImageInfo) (Probe, error) {
-	args, err := r.probeCommand(spec, api.ProbeTypeLiveness)
-	if err != nil {
-		return nil, err
-	}
+	args := r.probeCommand(spec, api.ProbeTypeLiveness)
 
 	cmdProbeConfig := &probes.CMDProbeConfig{
 		Command: args,
@@ -283,10 +273,7 @@ func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, gr
 
 func (r *Resources) probeBuilderStartupCoreOperator(spec api.DeploymentSpec, group api.ServerGroup,
 	image api.ImageInfo) (Probe, error) {
-	args, err := r.probeCommand(spec, api.ProbeTypeStartUp)
-	if err != nil {
-		return nil, err
-	}
+	args := r.probeCommand(spec, api.ProbeTypeStartUp)
 
 	retries, periodSeconds := getProbeRetries(group)
 	if IsServerProgressAvailable(group, image) {
@@ -399,10 +386,7 @@ func (r *Resources) probeBuilderReadinessCoreSelect() probeBuilder {
 }
 
 func (r *Resources) probeBuilderReadinessCoreOperator(spec api.DeploymentSpec, _ api.ServerGroup, _ api.ImageInfo) (Probe, error) {
-	args, err := r.probeCommand(spec, api.ProbeTypeReadiness)
-	if err != nil {
-		return nil, err
-	}
+	args := r.probeCommand(spec, api.ProbeTypeReadiness)
 
 	return &probes.CMDProbeConfig{
 		Command:             args,

pkg/deployment/rotation/arangod_containers.go

@@ -32,6 +32,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/topology"
 	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
 const (
@@ -61,7 +62,7 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
 
 					g := podContainerFuncGenerator(ds, g, ac, bc)
 
-					if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts)); err != nil {
+					if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts), g(compareServerContainerProbes)); err != nil {
 						log.Err(err).Msg("Error while getting pod diff")
 						return SkippedRotation, nil, err
 					} else {
@@ -104,11 +105,6 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
 						bc.Ports = ac.Ports
 						mode = mode.And(SilentRotation)
 					}
-
-					if !areProbesEqual(ac.StartupProbe, bc.StartupProbe) {
-						bc.StartupProbe = ac.StartupProbe
-						mode = mode.And(SilentRotation)
-					}
 				} else {
 					if ac.Image != bc.Image {
 						// Image changed
@@ -252,6 +248,18 @@ func areProbesEqual(a, b *core.Probe) bool {
 	return equality.Semantic.DeepEqual(a, b)
 }
 
+func isManagedProbe(a, b *core.Probe) bool {
+	if a.Exec == nil || b.Exec == nil {
+		return false
+	}
+
+	if len(a.Exec.Command) == 0 || len(b.Exec.Command) == 0 {
+		return false
+	}
+
+	return a.Exec.Command[0] == k8sutil.LifecycleBinary() && b.Exec.Command[0] == k8sutil.LifecycleBinary()
+}
+
 // areEnvsFromEqual returns true when environment variables from source are the same after filtering.
 func areEnvsFromEqual(a, b []core.EnvFromSource, rules ...func(a, b map[string]core.EnvFromSource) (map[string]core.EnvFromSource, map[string]core.EnvFromSource)) bool {
 	am := createEnvsFromMap(a)

pkg/deployment/rotation/arangod_probes.go

@@ -0,0 +1,65 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import (
+	core "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
+	return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
+		if !areProbesEqual(spec.StartupProbe, status.StartupProbe) {
+			status.StartupProbe = spec.StartupProbe
+			mode = mode.And(SilentRotation)
+		}
+
+		if !areProbesEqual(spec.ReadinessProbe, status.ReadinessProbe) {
+			if isManagedProbe(spec.ReadinessProbe, status.ReadinessProbe) {
+				q := status.ReadinessProbe.DeepCopy()
+
+				q.Exec = spec.ReadinessProbe.Exec.DeepCopy()
+
+				if equality.Semantic.DeepDerivative(spec.ReadinessProbe, q) {
+					status.ReadinessProbe = spec.ReadinessProbe
+					mode = mode.And(SilentRotation)
+				}
+			}
+		}
+
+		if !areProbesEqual(spec.LivenessProbe, status.LivenessProbe) {
+			if isManagedProbe(spec.LivenessProbe, status.LivenessProbe) {
+				q := status.LivenessProbe.DeepCopy()
+
+				q.Exec = spec.LivenessProbe.Exec.DeepCopy()
+
+				if equality.Semantic.DeepDerivative(spec.LivenessProbe, q) {
+					status.LivenessProbe = spec.LivenessProbe
+					mode = mode.And(SilentRotation)
+				}
+			}
+		}
+
+		return
+	}
+}

pkg/deployment/rotation/compare.go

@@ -23,6 +23,7 @@ package rotation
 import (
 	"encoding/json"
 
+	jd "github.com/josephburnett/jd/lib"
 	"github.com/rs/zerolog/log"
 	core "k8s.io/api/core/v1"
 
@@ -114,14 +115,27 @@ func compare(deploymentSpec api.DeploymentSpec, member api.MemberStatus, group a
 	}
 
 	if spec.RotationNeeded(newStatus) {
-		specData, _ := json.Marshal(spec)
-		statusData, _ := json.Marshal(newStatus)
-
-		log.Info().Str("before", spec.PodSpecChecksum).
-			Str("id", member.ID).
-			Str("spec", string(specData)).
-			Str("status", string(statusData)).
-			Msg("Pod needs rotation - templates does not match")
+		line := logger.Str("id", member.ID)
+
+		specBytes, errA := json.Marshal(spec.PodSpec)
+		if errA == nil {
+			line = line.Str("spec", string(specBytes))
+		}
+
+		statusBytes, errB := json.Marshal(newStatus.PodSpec)
+		if errB == nil {
+			line = line.Str("status", string(statusBytes))
+		}
+
+		if errA == nil && errB == nil {
+			if specData, err := jd.ReadJsonString(string(specBytes)); err == nil && specData != nil {
+				if statusData, err := jd.ReadJsonString(string(statusBytes)); err == nil && statusData != nil {
+					line = line.Str("diff", specData.Diff(statusData).Render())
+				}
+			}
+		}
+
+		line.Info("Pod needs rotation - templates does not match")
 
 		return GracefulRotation, nil, nil
 	}

pkg/deployment/rotation/logger.go

@@ -0,0 +1,27 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import "github.com/arangodb/kube-arangodb/pkg/logging"
+
+var (
+	logger = logging.Global().RegisterAndGetLogger("pod_compare", logging.Info)
+)