Merge pull request #148 from arangodb/feature/replication-status

Replication shard status in ArangoDeploymentReplication status

Author: ewoutp

main.go

@@ -160,7 +160,7 @@ func cmdMainRun(cmd *cobra.Command, args []string) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/health", livenessProbe.LivenessHandler)
 	mux.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler)
-	mux.HandleFunc("/ready/deployment-replications", deploymentReplicationProbe.ReadyHandler)
+	mux.HandleFunc("/ready/deployment-replication", deploymentReplicationProbe.ReadyHandler)
 	mux.HandleFunc("/ready/storage", storageProbe.ReadyHandler)
 	mux.Handle("/metrics", prometheus.Handler())
 	listenAddr := net.JoinHostPort(serverOptions.host, strconv.Itoa(serverOptions.port))

pkg/apis/replication/v1alpha/collection_status.go

@@ -0,0 +1,32 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+// CollectionStatus contains the status of a single collection.
+type CollectionStatus struct {
+	// Name of the collection
+	Name string `json:"name"`
+	// Replication status per shard.
+	// The list is ordered by shard index (0..noShards-1)
+	Shards []ShardStatus `json:"shards,omitempty"`
+}

pkg/apis/replication/v1alpha/database_status.go

@@ -0,0 +1,32 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+// DatabaseStatus contains the status of a single database.
+type DatabaseStatus struct {
+	// Name of the database
+	Name string `json:"name"`
+	// Collections holds the replication status of each collection in the database.
+	// List is ordered by name of the collection.
+	Collections []CollectionStatus `json:"collections,omitempty"`
+}

pkg/apis/replication/v1alpha/endpoint_status.go

@@ -0,0 +1,30 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+// EndpointStatus contains the status of either the source or destination endpoint.
+type EndpointStatus struct {
+	// Databases holds the replication status of all databases from the point of view of this endpoint.
+	// List is ordered by name of the database.
+	Databases []DatabaseStatus `json:"databases,omitempty"`
+}

pkg/apis/replication/v1alpha/replication_phase.go

@@ -28,9 +28,6 @@ type DeploymentReplicationPhase string
 const (
 	// DeploymentReplicationPhaseNone indicates that the phase is not set yet
 	DeploymentReplicationPhaseNone DeploymentReplicationPhase = ""
-	// DeploymentReplicationPhaseRunning indicates that the deployment replication is under control of the
-	// ArangoDeploymentReplication operator.
-	DeploymentReplicationPhaseRunning DeploymentReplicationPhase = "Running"
 	// DeploymentReplicationPhaseFailed indicates that a deployment replication is in a failed state
 	// from which automatic recovery is impossible. Inspect `Reason` for more info.
 	DeploymentReplicationPhaseFailed DeploymentReplicationPhase = "Failed"

pkg/apis/replication/v1alpha/replication_status.go

@@ -26,10 +26,19 @@ package v1alpha
 // an ArangoDeploymentReplication.
 type DeploymentReplicationStatus struct {
 	// Phase holds the current lifetime phase of the deployment replication
-	Phase DeploymentReplicationPhase `json:"phase"`
+	Phase DeploymentReplicationPhase `json:"phase,omitempty"`
 	// Reason contains a human readable reason for reaching the current phase (can be empty)
 	Reason string `json:"reason,omitempty"` // Reason for current phase
 
 	// Conditions specific to the entire deployment replication
 	Conditions ConditionList `json:"conditions,omitempty"`
+
+	// Source contains the detailed status of the source endpoint
+	Source EndpointStatus `json:"source"`
+	// Destination contains the detailed status of the destination endpoint
+	Destination EndpointStatus `json:"destination"`
+
+	// CancelFailures records the number of times that the configuration was canceled
+	// which resulted in an error.
+	CancelFailures int `json:"cancel-failures,omitempty"`
 }

pkg/apis/replication/v1alpha/shard_status.go

@@ -0,0 +1,28 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+// ShardStatus contains the status of a single shard.
+type ShardStatus struct {
+	Status string `json:"status"`
+}

pkg/apis/replication/v1alpha/zz_generated.deepcopy.go

@@ -89,6 +89,27 @@ func (in *ArangoDeploymentReplicationList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CollectionStatus) DeepCopyInto(out *CollectionStatus) {
+	*out = *in
+	if in.Shards != nil {
+		in, out := &in.Shards, &out.Shards
+		*out = make([]ShardStatus, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollectionStatus.
+func (in *CollectionStatus) DeepCopy() *CollectionStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(CollectionStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Condition) DeepCopyInto(out *Condition) {
 	*out = *in
@@ -107,6 +128,29 @@ func (in *Condition) DeepCopy() *Condition {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DatabaseStatus) DeepCopyInto(out *DatabaseStatus) {
+	*out = *in
+	if in.Collections != nil {
+		in, out := &in.Collections, &out.Collections
+		*out = make([]CollectionStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatabaseStatus.
+func (in *DatabaseStatus) DeepCopy() *DatabaseStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DatabaseStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentReplicationSpec) DeepCopyInto(out *DeploymentReplicationSpec) {
 	*out = *in
@@ -135,6 +179,8 @@ func (in *DeploymentReplicationStatus) DeepCopyInto(out *DeploymentReplicationSt
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.Source.DeepCopyInto(&out.Source)
+	in.Destination.DeepCopyInto(&out.Destination)
 	return
 }
 
@@ -214,6 +260,29 @@ func (in *EndpointSpec) DeepCopy() *EndpointSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EndpointStatus) DeepCopyInto(out *EndpointStatus) {
+	*out = *in
+	if in.Databases != nil {
+		in, out := &in.Databases, &out.Databases
+		*out = make([]DatabaseStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointStatus.
+func (in *EndpointStatus) DeepCopy() *EndpointStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(EndpointStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *EndpointTLSSpec) DeepCopyInto(out *EndpointTLSSpec) {
 	*out = *in
@@ -238,3 +307,19 @@ func (in *EndpointTLSSpec) DeepCopy() *EndpointTLSSpec {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ShardStatus) DeepCopyInto(out *ShardStatus) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardStatus.
+func (in *ShardStatus) DeepCopy() *ShardStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ShardStatus)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/replication/finalizers.go

@@ -37,6 +37,10 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
+const (
+	maxCancelFailures = 5 // After this amount of failed cancel-synchronization attempts, the operator switch to abort-sychronization.
+)
+
 // addFinalizers adds a stop-sync finalizer to the api object when needed.
 func (dr *DeploymentReplication) addFinalizers() error {
 	apiObject := dr.apiObject
@@ -92,7 +96,7 @@ func (dr *DeploymentReplication) inspectFinalizerDeplReplStopSync(ctx context.Co
 	}
 
 	// Inspect deployment deletion state in source
-	abort := false
+	abort := dr.status.CancelFailures > maxCancelFailures
 	depls := dr.deps.CRCli.DatabaseV1alpha().ArangoDeployments(p.GetNamespace())
 	if name := p.Spec.Source.GetDeploymentName(); name != "" {
 		depl, err := depls.Get(name, metav1.GetOptions{})
@@ -150,6 +154,10 @@ func (dr *DeploymentReplication) inspectFinalizerDeplReplStopSync(ctx context.Co
 		_, err = destClient.Master().CancelSynchronization(ctx, req)
 		if err != nil && !client.IsPreconditionFailed(err) {
 			log.Warn().Err(err).Bool("abort", abort).Msg("Failed to stop synchronization")
+			dr.status.CancelFailures++
+			if err := dr.updateCRStatus(); err != nil {
+				log.Warn().Err(err).Msg("Failed to update status to reflect cancel-failures increment")
+			}
 			return maskAny(err)
 		}
 		return nil