Added renewal of deployment TLS CA certificate

Author: ewoutp

pkg/apis/deployment/v1alpha/plan.go

@@ -47,6 +47,8 @@ const (
 	ActionTypeWaitForMemberUp ActionType = "WaitForMemberUp"
 	// ActionTypeRenewTLSCertificate causes the TLS certificate of a member to be renewed.
 	ActionTypeRenewTLSCertificate ActionType = "RenewTLSCertificate"
+	// ActionTypeRenewTLSCACertificate causes the TLS CA certificate of the entire deployment to be renewed.
+	ActionTypeRenewTLSCACertificate ActionType = "RenewTLSCACertificate"
 )
 
 // Action represents a single action to be taken to update a deployment.

pkg/deployment/access_package.go

@@ -108,7 +108,7 @@ func (d *Deployment) ensureAccessPackage(apSecretName string) error {
 
 	// Fetch client authentication CA
 	clientAuthSecretName := spec.Sync.Authentication.GetClientCASecretName()
-	clientAuthCert, clientAuthKey, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), clientAuthSecretName, ns)
+	clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), clientAuthSecretName, ns, nil)
 	if err != nil {
 		log.Debug().Err(err).Msg("Failed to get client-auth CA secret")
 		return maskAny(err)

pkg/deployment/context_impl.go

@@ -237,6 +237,25 @@ func (d *Deployment) CleanupPod(p v1.Pod) error {
 	return nil
 }
 
+// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+// of the deployment. If the pod does not exist, the error is ignored.
+func (d *Deployment) RemovePodFinalizers(podName string) error {
+	log := d.deps.Log
+	ns := d.GetNamespace()
+	kubecli := d.deps.KubeCli
+	p, err := kubecli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
+	if err != nil {
+		if k8sutil.IsNotFound(err) {
+			return nil
+		}
+		return maskAny(err)
+	}
+	if err := k8sutil.RemovePodFinalizers(log, d.deps.KubeCli, p, p.GetFinalizers(), true); err != nil {
+		return maskAny(err)
+	}
+	return nil
+}
+
 // DeletePvc deletes a persistent volume claim with given name in the namespace
 // of the deployment. If the pvc does not exist, the error is ignored.
 func (d *Deployment) DeletePvc(pvcName string) error {
@@ -307,3 +326,26 @@ func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberSt
 	}
 	return nil
 }
+
+// GetTLSCA returns the TLS CA certificate in the secret with given name.
+// Returns: publicKey, privateKey, ownerByDeployment, error
+func (d *Deployment) GetTLSCA(secretName string) (string, string, bool, error) {
+	ns := d.apiObject.GetNamespace()
+	owner := d.apiObject.AsOwner()
+	cert, priv, isOwned, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), secretName, ns, &owner)
+	if err != nil {
+		return "", "", false, maskAny(err)
+	}
+	return cert, priv, isOwned, nil
+
+}
+
+// DeleteSecret removes the Secret with given name.
+// If the secret does not exist, the error is ignored.
+func (d *Deployment) DeleteSecret(secretName string) error {
+	ns := d.apiObject.GetNamespace()
+	if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
+		return maskAny(err)
+	}
+	return nil
+}

pkg/deployment/reconcile/action_context.go

@@ -70,9 +70,14 @@ type ActionContext interface {
 	// DeletePvc deletes a persistent volume claim with given name in the namespace
 	// of the deployment. If the pvc does not exist, the error is ignored.
 	DeletePvc(pvcName string) error
+	// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+	// of the deployment. If the pod does not exist, the error is ignored.
+	RemovePodFinalizers(podName string) error
 	// DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member.
 	// If the secret does not exist, the error is ignored.
 	DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error
+	// DeleteTLSCASecret removes the Secret containing the TLS CA certificate.
+	DeleteTLSCASecret() error
 }
 
 // newActionContext creates a new ActionContext implementation.
@@ -212,6 +217,15 @@ func (ac *actionContext) DeletePvc(pvcName string) error {
 	return nil
 }
 
+// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+// of the deployment. If the pod does not exist, the error is ignored.
+func (ac *actionContext) RemovePodFinalizers(podName string) error {
+	if err := ac.context.RemovePodFinalizers(podName); err != nil {
+		return maskAny(err)
+	}
+	return nil
+}
+
 // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member.
 // If the secret does not exist, the error is ignored.
 func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error {
@@ -220,3 +234,28 @@ func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.Memb
 	}
 	return nil
 }
+
+// DeleteTLSCASecret removes the Secret containing the TLS CA certificate.
+func (ac *actionContext) DeleteTLSCASecret() error {
+	spec := ac.context.GetSpec().TLS
+	if !spec.IsSecure() {
+		return nil
+	}
+	secretName := spec.GetCASecretName()
+	if secretName == "" {
+		return nil
+	}
+	// Remove secret hash, since it is going to change
+	status, lastVersion := ac.context.GetStatus()
+	if status.SecretHashes != nil {
+		status.SecretHashes.TLSCA = ""
+		if err := ac.context.UpdateStatus(status, lastVersion); err != nil {
+			return maskAny(err)
+		}
+	}
+	// Do delete the secret
+	if err := ac.context.DeleteSecret(secretName); err != nil {
+		return maskAny(err)
+	}
+	return nil
+}

pkg/deployment/reconcile/action_renew_tls_ca_certificate.go

@@ -0,0 +1,71 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package reconcile
+
+import (
+	"context"
+	"time"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+	"github.com/rs/zerolog"
+)
+
+// NewRenewTLSCACertificateAction creates a new Action that implements the given
+// planned RenewTLSCACertificate action.
+func NewRenewTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	return &renewTLSCACertificateAction{
+		log:       log,
+		action:    action,
+		actionCtx: actionCtx,
+	}
+}
+
+// renewTLSCACertificateAction implements a RenewTLSCACertificate action.
+type renewTLSCACertificateAction struct {
+	log       zerolog.Logger
+	action    api.Action
+	actionCtx ActionContext
+}
+
+// Start performs the start of the action.
+// Returns true if the action is completely finished, false in case
+// the start time needs to be recorded and a ready condition needs to be checked.
+func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) {
+	// Just delete the secret.
+	// It will be re-created.
+	if err := a.actionCtx.DeleteTLSCASecret(); err != nil {
+		return false, maskAny(err)
+	}
+	return true, nil
+}
+
+// CheckProgress checks the progress of the action.
+// Returns true if the action is completely finished, false otherwise.
+func (a *renewTLSCACertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) {
+	return true, false, nil
+}
+
+// Timeout returns the amount of time after which this action will timeout.
+func (a *renewTLSCACertificateAction) Timeout() time.Duration {
+	return renewTLSCACertificateTimeout
+}

pkg/deployment/reconcile/action_rotate_member.go

@@ -57,6 +57,10 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) {
 	if !ok {
 		log.Error().Msg("No such member")
 	}
+	// Remove finalizers, so Kubernetes will quickly terminate the pod
+	if err := a.actionCtx.RemovePodFinalizers(m.PodName); err != nil {
+		return false, maskAny(err)
+	}
 	if group.IsArangod() {
 		// Invoke shutdown endpoint
 		c, err := a.actionCtx.GetServerClient(ctx, group, a.action.MemberID)

pkg/deployment/reconcile/context.go

@@ -69,6 +69,9 @@ type Context interface {
 	// DeletePvc deletes a persistent volume claim with given name in the namespace
 	// of the deployment. If the pvc does not exist, the error is ignored.
 	DeletePvc(pvcName string) error
+	// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+	// of the deployment. If the pod does not exist, the error is ignored.
+	RemovePodFinalizers(podName string) error
 	// GetOwnedPods returns a list of all pods owned by the deployment.
 	GetOwnedPods() ([]v1.Pod, error)
 	// GetTLSKeyfile returns the keyfile encoded TLS certificate+key for
@@ -77,4 +80,10 @@ type Context interface {
 	// DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member.
 	// If the secret does not exist, the error is ignored.
 	DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error
+	// GetTLSCA returns the TLS CA certificate in the secret with given name.
+	// Returns: publicKey, privateKey, ownerByDeployment, error
+	GetTLSCA(secretName string) (string, string, bool, error)
+	// DeleteSecret removes the Secret with given name.
+	// If the secret does not exist, the error is ignored.
+	DeleteSecret(secretName string) error
 }

pkg/deployment/reconcile/plan_builder.go

@@ -23,10 +23,6 @@
 package reconcile
 
 import (
-	"crypto/x509"
-	"encoding/pem"
-	"time"
-
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"k8s.io/api/core/v1"
@@ -58,7 +54,7 @@ func (d *Reconciler) CreatePlan() error {
 	apiObject := d.context.GetAPIObject()
 	spec := d.context.GetSpec()
 	status, lastVersion := d.context.GetStatus()
-	newPlan, changed := createPlan(d.log, apiObject, status.Plan, spec, status, pods, d.context.GetTLSKeyfile)
+	newPlan, changed := createPlan(d.log, apiObject, status.Plan, spec, status, pods, d.context.GetTLSKeyfile, d.context.GetTLSCA)
 
 	// If not change, we're done
 	if !changed {
@@ -83,7 +79,8 @@ func (d *Reconciler) CreatePlan() error {
 func createPlan(log zerolog.Logger, apiObject metav1.Object,
 	currentPlan api.Plan, spec api.DeploymentSpec,
 	status api.DeploymentStatus, pods []v1.Pod,
-	getTLSKeyfile func(group api.ServerGroup, member api.MemberStatus) (string, error)) (api.Plan, bool) {
+	getTLSKeyfile func(group api.ServerGroup, member api.MemberStatus) (string, error),
+	getTLSCA func(string) (string, string, bool, error)) (api.Plan, bool) {
 	if len(currentPlan) > 0 {
 		// Plan already exists, complete that first
 		return currentPlan, false
@@ -178,41 +175,14 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object,
 		})
 	}
 
+	// Check for the need to rotate TLS CA certificate and all members
+	if len(plan) == 0 {
+		plan = createRotateTLSCAPlan(log, spec, status, getTLSCA)
+	}
+
 	// Check for the need to rotate TLS certificate of a members
-	if len(plan) == 0 && spec.TLS.IsSecure() {
-		status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
-			for _, m := range members {
-				if len(plan) > 0 {
-					// Only 1 change at a time
-					continue
-				}
-				if m.Phase != api.MemberPhaseCreated {
-					// Only make changes when phase is created
-					continue
-				}
-				if group == api.ServerGroupSyncWorkers {
-					// SyncWorkers have no externally created TLS keyfile
-					continue
-				}
-				// Load keyfile
-				keyfile, err := getTLSKeyfile(group, m)
-				if err != nil {
-					log.Warn().Err(err).
-						Str("role", group.AsRole()).
-						Str("id", m.ID).
-						Msg("Failed to get TLS secret")
-					continue
-				}
-				renewalNeeded := tlsKeyfileNeedsRenewal(log, keyfile)
-				if renewalNeeded {
-					plan = append(append(plan,
-						api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID)),
-						createRotateMemberPlan(log, m, group, "TLS certificate renewal")...,
-					)
-				}
-			}
-			return nil
-		})
+	if len(plan) == 0 {
+		plan = createRotateTLSServerCertificatePlan(log, spec, status, getTLSKeyfile)
 	}
 
 	// Return plan
@@ -304,44 +274,6 @@ func normalizeServiceAccountName(name string) string {
 	return ""
 }
 
-// tlsKeyfileNeedsRenewal decides if the certificate in the given keyfile
-// should be renewed.
-func tlsKeyfileNeedsRenewal(log zerolog.Logger, keyfile string) bool {
-	raw := []byte(keyfile)
-	for {
-		var derBlock *pem.Block
-		derBlock, raw = pem.Decode(raw)
-		if derBlock == nil {
-			break
-		}
-		if derBlock.Type == "CERTIFICATE" {
-			cert, err := x509.ParseCertificate(derBlock.Bytes)
-			if err != nil {
-				// We do not understand the certificate, let's renew it
-				log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it")
-				return true
-			}
-			if cert.IsCA {
-				// Only look at the server certificate, not CA or intermediate
-				continue
-			}
-			// Check expiration date. Renewal at 2/3 of lifetime.
-			ttl := cert.NotAfter.Sub(cert.NotBefore)
-			expirationDate := cert.NotBefore.Add((ttl / 3) * 2)
-			if expirationDate.Before(time.Now()) {
-				// We should renew now
-				log.Debug().
-					Str("not-before", cert.NotBefore.String()).
-					Str("not-after", cert.NotAfter.String()).
-					Str("expiration-date", expirationDate.String()).
-					Msg("TLS certificate renewal needed")
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // createScalePlan creates a scaling plan for a single server group
 func createScalePlan(log zerolog.Logger, members api.MemberStatusList, group api.ServerGroup, count int) api.Plan {
 	var plan api.Plan