Feature/restart pods on sidecar changes (#420)

* added side car changes to pod rotation
* change log
* go fmt
* sidecar test

Author: kvahed

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [0.3.13]() (XXXX-XX-XX)
+- Added side car changed to pod rotation criterium
 - Added ArangoDB version and image id to member status
 
 ## [0.3.12](https://github.com/arangodb/kube-arangodb/tree/0.3.12) (2019-07-04)

docs/Manual/Tutorials/Kubernetes/bare-metal.md

@@ -220,7 +220,7 @@ kubectl get all --all-namespaces
 - Attach `tiller` to proper role
 
   ```
-kubectl create clusterrolebinding tiller-cluster-rule \
+  kubectl create clusterrolebinding tiller-cluster-rule \
 	--clusterrole=cluster-admin --serviceaccount=kube-system:tiller
   ```
   ```

pkg/apis/deployment/v1alpha/deployment_status.go

@@ -22,7 +22,9 @@
 
 package v1alpha
 
-import "github.com/arangodb/kube-arangodb/pkg/util"
+import (
+	"github.com/arangodb/kube-arangodb/pkg/util"
+)
 
 // DeploymentStatus contains the status part of a Cluster resource.
 type DeploymentStatus struct {

pkg/apis/deployment/v1alpha/member_status.go

@@ -23,6 +23,7 @@
 package v1alpha
 
 import (
+	"reflect"
 	"time"
 
 	driver "github.com/arangodb/go-driver"
@@ -54,6 +55,8 @@ type MemberStatus struct {
 	IsInitialized bool `json:"initialized"`
 	// CleanoutJobID holds the ID of the agency job for cleaning out this server
 	CleanoutJobID string `json:"cleanout-job-id,omitempty"`
+	// SideCarSpecs contains list of specifications specified for side cars
+	SideCarSpecs map[string]v1.Container
 	// ArangoVersion holds the ArangoDB version in member
 	ArangoVersion driver.Version `json:"arango-version,omitempty"`
 	//ImageId holds the members ArangoDB image ID
@@ -70,6 +73,7 @@ func (s MemberStatus) Equal(other MemberStatus) bool {
 		s.Conditions.Equal(other.Conditions) &&
 		s.IsInitialized == other.IsInitialized &&
 		s.CleanoutJobID == other.CleanoutJobID &&
+		reflect.DeepEqual(s.SideCarSpecs, other.SideCarSpecs) &&
 		s.ArangoVersion == other.ArangoVersion &&
 		s.ImageID == other.ImageID
 }

pkg/deployment/reconcile/action_context.go

@@ -88,6 +88,9 @@ type ActionContext interface {
 	GetDeploymentHealth() (driver.ClusterHealth, error)
 	// InvalidateSyncStatus resets the sync state to false and triggers an inspection
 	InvalidateSyncStatus()
+	// GetSpec returns a copy of the spec
+	GetSpec() api.DeploymentSpec
+
 }
 
 // newActionContext creates a new ActionContext implementation.
@@ -109,6 +112,10 @@ func (ac *actionContext) GetMode() api.DeploymentMode {
 	return ac.context.GetSpec().GetMode()
 }
 
+func (ac *actionContext) GetSpec() api.DeploymentSpec {
+	return ac.context.GetSpec()
+}
+
 // GetDeploymentHealth returns a copy of the latest known state of cluster health
 func (ac *actionContext) GetDeploymentHealth() (driver.ClusterHealth, error) {
 	return ac.context.GetDeploymentHealth()

pkg/deployment/reconcile/action_rotate_member.go

@@ -26,6 +26,7 @@ import (
 	"context"
 	"time"
 
+	"k8s.io/api/core/v1"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
 	"github.com/rs/zerolog"
 )
@@ -89,6 +90,7 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) {
 	}
 	// Update status
 	m.Phase = api.MemberPhaseRotating
+
 	if err := a.actionCtx.UpdateMember(m); err != nil {
 		return false, maskAny(err)
 	}
@@ -117,6 +119,15 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err
 	m.Phase = api.MemberPhaseNone
 	m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations.
 	m.CleanoutJobID = ""
+
+	group := a.action.Group
+	var groupSpec = a.actionCtx.GetSpec().GetServerGroupSpec(group)
+	// Check for missing side cars in
+	m.SideCarSpecs = make(map[string]v1.Container)
+	for _, specSidecar := range groupSpec.GetSidecars() {
+		m.SideCarSpecs[specSidecar.Name] = *specSidecar.DeepCopy()
+	}
+
 	if err := a.actionCtx.UpdateMember(m); err != nil {
 		return false, false, maskAny(err)
 	}

pkg/deployment/reconcile/plan_builder.go

@@ -23,17 +23,18 @@
 package reconcile
 
 import (
+	"reflect"
 	"strings"
 
 	driver "github.com/arangodb/go-driver"
 	upgraderules "github.com/arangodb/go-upgrade-rules"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
-	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	v1 "k8s.io/api/core/v1"
 )
 
 // upgradeDecision is the result of an upgrade check.
@@ -394,9 +395,106 @@ func podNeedsRotation(log zerolog.Logger, p v1.Pod, apiObject metav1.Object, spe
 		return true, "Resource Requirements changed"
 	}
 
+	var memberStatus, _, _ = status.Members.MemberStatusByPodName(p.GetName())
+	if memberStatus.SideCarSpecs == nil {
+		memberStatus.SideCarSpecs = make(map[string]v1.Container)
+	}
+
+	// Check for missing side cars in
+	for _, specSidecar := range groupSpec.GetSidecars() {
+		var stateSidecar v1.Container
+		if stateSidecar, found = memberStatus.SideCarSpecs[specSidecar.Name]; !found {
+			return true, "Sidecar " + specSidecar.Name + " not found in running pod " + p.GetName()
+		}
+		if sideCarRequireRotation(specSidecar.DeepCopy(), &stateSidecar) {
+			return true, "Sidecar " + specSidecar.Name + " requires rotation"
+		}
+	}
+
+	for name := range memberStatus.SideCarSpecs {
+		var found = false
+		for _, specSidecar := range groupSpec.GetSidecars() {
+			if name == specSidecar.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return true, "Sidecar " + name + " no longer in specification"
+		}
+	}
+
 	return false, ""
 }
 
+// sideCarRequireRotation checks if side car requires rotation including default parameters
+func sideCarRequireRotation(wanted, given *v1.Container) bool {
+	if !reflect.DeepEqual(wanted.Args, given.Args) {
+		return true
+	}
+	if !reflect.DeepEqual(wanted.Command, given.Command) {
+		return true
+	}
+	if !reflect.DeepEqual(wanted.Env, given.Env) {
+		return true
+	}
+	if !reflect.DeepEqual(wanted.EnvFrom, given.EnvFrom) {
+		return true
+	}
+	if wanted.Image != given.Image {
+		return true
+	}
+	if wanted.ImagePullPolicy != given.ImagePullPolicy {
+		if wanted.ImagePullPolicy != "Always" || !strings.HasSuffix(given.Image, ":latest") {
+			return true
+		}
+	}
+	if wanted.Lifecycle != given.Lifecycle {
+		return true
+	}
+	if wanted.LivenessProbe != given.LivenessProbe {
+		return true
+	}
+	if !reflect.DeepEqual(wanted.Ports, given.Ports) {
+		return true
+	}
+	if wanted.ReadinessProbe != given.ReadinessProbe {
+		return true
+	}
+	if !reflect.DeepEqual(wanted.Resources, given.Resources) {
+		return true
+	}
+	if wanted.SecurityContext != given.SecurityContext {
+		return true
+	}
+	if wanted.Stdin != given.Stdin {
+		return true
+	}
+	if wanted.StdinOnce != given.StdinOnce {
+		return true
+	}
+	if wanted.TerminationMessagePath != given.TerminationMessagePath {
+		return true
+	}
+	if wanted.TerminationMessagePolicy != given.TerminationMessagePolicy {
+		return true
+	}
+	if wanted.TTY != given.TTY {
+		return true
+	}
+	if !reflect.DeepEqual(wanted.VolumeDevices, given.VolumeDevices) {
+		return true
+	}
+	if !reflect.DeepEqual(wanted.VolumeMounts, given.VolumeMounts) {
+		return true
+	}
+	if wanted.WorkingDir != given.WorkingDir {
+		return true
+	}
+
+	return false
+}
+
 // resourcesRequireRotation returns true if the resource requirements have changed such that a rotation is required
 func resourcesRequireRotation(wanted, given v1.ResourceRequirements) bool {
 	checkList := func(wanted, given v1.ResourceList) bool {

pkg/deployment/resources/member_cleanup.go

@@ -71,6 +71,7 @@ func (r *Resources) cleanupRemovedClusterMembers() error {
 	r.health.mutex.Unlock()
 
 	// Only accept recent cluster health values
+
 	healthAge := time.Since(ts)
 	if healthAge > maxClusterHealthAge {
 		log.Info().Dur("age", healthAge).Msg("Cleanup longer than max cluster health. Exiting")

tests/load_balancer_source_ranges_test.go

@@ -108,7 +108,7 @@ func TestLoadBalancingSourceRanges(t *testing.T) {
 		}
 		t.Logf("Service %s cannot be found, waiting for some time...", eaServiceName)
 		time.Sleep(time.Second)
-		counter += 1
+		counter++
 		if counter >= 60 {
 			t.Fatalf("Could not find service %s within 60 seconds, giving up.", eaServiceName)
 		}
@@ -149,7 +149,7 @@ func TestLoadBalancingSourceRanges(t *testing.T) {
 			}
 		}
 		t.Logf("Service %s cannot be found, waiting for some more time...", eaServiceName)
-		counter += 1
+		counter++
 		if counter >= 60 {
 			t.Fatalf("Could not find changed service %s within 60 seconds, giving up.", eaServiceName)
 		}