Merge pull request #107 from arangodb/detect-missing-deployment

Quickly fail when deployment no longer exists

Author: ewoutp

pkg/deployment/deployment.go

@@ -25,6 +25,7 @@ package deployment
 import (
 	"fmt"
 	"reflect"
+	"sync/atomic"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -86,6 +87,7 @@ type Deployment struct {
 
 	eventCh chan *deploymentEvent
 	stopCh  chan struct{}
+	stopped int32
 
 	eventsCli corev1.EventInterface
 
@@ -154,7 +156,9 @@ func (d *Deployment) Update(apiObject *api.ArangoDeployment) {
 // Called when the deployment was deleted by the user.
 func (d *Deployment) Delete() {
 	d.deps.Log.Info().Msg("deployment is deleted by user")
-	close(d.stopCh)
+	if atomic.CompareAndSwapInt32(&d.stopped, 0, 1) {
+		close(d.stopCh)
+	}
 }
 
 // send given event into the deployment event queue.

pkg/deployment/deployment_inspector.go

@@ -28,6 +28,7 @@ import (
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // inspectDeployment inspects the entire deployment, creates
@@ -44,6 +45,14 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration
 	hasError := false
 	ctx := context.Background()
 
+	// Check deployment still exists
+	if _, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) {
+		// Deployment is gone
+		log.Info().Msg("Deployment is gone")
+		d.Delete()
+		return nextInterval
+	}
+
 	// Is the deployment in failed state, if so, give up.
 	if d.status.Phase == api.DeploymentPhaseFailed {
 		log.Debug().Msg("Deployment is in Failed state.")

pkg/storage/local_storage.go

@@ -26,6 +26,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"sync/atomic"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -89,6 +90,7 @@ type LocalStorage struct {
 
 	eventCh chan *localStorageEvent
 	stopCh  chan struct{}
+	stopped int32
 
 	eventsCli corev1.EventInterface
 
@@ -136,7 +138,9 @@ func (ls *LocalStorage) Update(apiObject *api.ArangoLocalStorage) {
 // Called when the local storage was deleted by the user.
 func (ls *LocalStorage) Delete() {
 	ls.deps.Log.Info().Msg("local storage is deleted by user")
-	close(ls.stopCh)
+	if atomic.CompareAndSwapInt32(&ls.stopped, 0, 1) {
+		close(ls.stopCh)
+	}
 }
 
 // send given event into the local storage event queue.