Endpoint: fix terminal condition showing up as recoverable (#18)

Author: surajkota

generator.yaml

@@ -84,6 +84,8 @@ resources:
         - MalformedQueryString
         - InvalidAction
         - UnrecognizedClientException
+        # Custom error
+        - EndpointUpdateError
     hooks:
       sdk_create_post_set_output:
         code: rm.customSetOutput(r, aws.String(svcsdk.EndpointStatusCreating), ko)

pkg/resource/endpoint/custom_set_output.go

@@ -85,14 +85,10 @@ func (rm *resourceManager) customSetOutput(
 	}
 
 	var resourceSyncedCondition *ackv1alpha1.Condition = nil
-	if ko.Status.Conditions == nil {
-		ko.Status.Conditions = []*ackv1alpha1.Condition{}
-	} else {
-		for _, condition := range ko.Status.Conditions {
-			if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
-				resourceSyncedCondition = condition
-				break
-			}
+	for _, condition := range ko.Status.Conditions {
+		if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
+			resourceSyncedCondition = condition
+			break
 		}
 	}
 

pkg/resource/endpoint/custom_update_api.go

@@ -19,16 +19,16 @@ package endpoint
 import (
 	"context"
 	"errors"
-	"fmt"
 	"strings"
 
 	ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare"
 	"github.com/aws-controllers-k8s/runtime/pkg/requeue"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	svcsdk "github.com/aws/aws-sdk-go/service/sagemaker"
 )
 
 var (
-	FailUpdateError = fmt.Errorf("Unable to update Endpoint. Check FailureReason")
+	FailUpdateError = awserr.New("EndpointUpdateError", "unable to update endpoint. check FailureReason", nil)
 
 	FailureReasonInternalServiceErrorPrefix = "Request to service failed"
 )

pkg/resource/endpoint/custom_update_conditions.go

@@ -33,43 +33,32 @@ func (rm *resourceManager) customUpdateConditions(
 	err error,
 ) bool {
 	latestStatus := r.ko.Status.EndpointStatus
-	failureReason := r.ko.Status.FailureReason
 
-	if latestStatus == nil || failureReason == nil {
+	if latestStatus == nil || *latestStatus != svcsdk.EndpointStatusFailed {
 		return false
 	}
 	var terminalCondition *ackv1alpha1.Condition = nil
-	if ko.Status.Conditions == nil {
-		ko.Status.Conditions = []*ackv1alpha1.Condition{}
-	} else {
-		for _, condition := range ko.Status.Conditions {
-			if condition.Type == ackv1alpha1.ConditionTypeTerminal {
-				terminalCondition = condition
-				break
-			}
-		}
-		if terminalCondition != nil && terminalCondition.Status == corev1.ConditionTrue {
-			// some other exception already put the resource in terminal condition
-			return false
+
+	for _, condition := range ko.Status.Conditions {
+		if condition.Type == ackv1alpha1.ConditionTypeTerminal {
+			terminalCondition = condition
+			break
 		}
 	}
+	if terminalCondition != nil && terminalCondition.Status == corev1.ConditionTrue {
+		// some other exception already put the resource in terminal condition
+		return false
+	}
 
-	if (err != nil && err == FailUpdateError) || (latestStatus != nil && *latestStatus == svcsdk.EndpointStatusFailed) {
-		// setting terminal condition since controller can no longer recover by retrying
-		if terminalCondition == nil {
-			terminalCondition = &ackv1alpha1.Condition{
-				Type: ackv1alpha1.ConditionTypeTerminal,
-			}
-			ko.Status.Conditions = append(ko.Status.Conditions, terminalCondition)
-		}
-		terminalCondition.Status = corev1.ConditionTrue
-		if *latestStatus == svcsdk.EndpointStatusFailed {
-			terminalCondition.Message = aws.String("Cannot update endpoint with Failed status")
-		} else {
-			terminalCondition.Message = aws.String(FailUpdateError.Error())
+	// setting terminal condition since controller can no longer recover by retrying
+	if terminalCondition == nil {
+		terminalCondition = &ackv1alpha1.Condition{
+			Type: ackv1alpha1.ConditionTypeTerminal,
 		}
-		return true
+		ko.Status.Conditions = append(ko.Status.Conditions, terminalCondition)
 	}
+	terminalCondition.Status = corev1.ConditionTrue
+	terminalCondition.Message = aws.String("endpoint status: Failed. check FailureReason")
 
-	return false
+	return true
 }

pkg/resource/endpoint/sdk.go

@@ -43,14 +43,10 @@ func (rm *resourceManager) customSetOutput(
 	}
 
 	var resourceSyncedCondition *ackv1alpha1.Condition = nil
-	if ko.Status.Conditions == nil {
-		ko.Status.Conditions = []*ackv1alpha1.Condition{}
-	} else {
-		for _, condition := range ko.Status.Conditions {
-			if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
-				resourceSyncedCondition = condition
-				break
-			}
+	for _, condition := range ko.Status.Conditions {
+		if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
+			resourceSyncedCondition = condition
+			break
 		}
 	}
 

pkg/resource/hyper_parameter_tuning_job/custom_set_output.go

@@ -43,14 +43,10 @@ func (rm *resourceManager) customSetOutput(
 	}
 
 	var resourceSyncedCondition *ackv1alpha1.Condition = nil
-	if ko.Status.Conditions == nil {
-		ko.Status.Conditions = []*ackv1alpha1.Condition{}
-	} else {
-		for _, condition := range ko.Status.Conditions {
-			if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
-				resourceSyncedCondition = condition
-				break
-			}
+	for _, condition := range ko.Status.Conditions {
+		if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
+			resourceSyncedCondition = condition
+			break
 		}
 	}
 

pkg/resource/processing_job/custom_set_output.go

@@ -73,14 +73,10 @@ func (rm *resourceManager) customSetOutput(
 	}
 
 	var resourceSyncedCondition *ackv1alpha1.Condition = nil
-	if ko.Status.Conditions == nil {
-		ko.Status.Conditions = []*ackv1alpha1.Condition{}
-	} else {
-		for _, condition := range ko.Status.Conditions {
-			if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
-				resourceSyncedCondition = condition
-				break
-			}
+	for _, condition := range ko.Status.Conditions {
+		if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
+			resourceSyncedCondition = condition
+			break
 		}
 	}
 

pkg/resource/training_job/custom_set_output.go

@@ -43,14 +43,10 @@ func (rm *resourceManager) customSetOutput(
 	}
 
 	var resourceSyncedCondition *ackv1alpha1.Condition = nil
-	if ko.Status.Conditions == nil {
-		ko.Status.Conditions = []*ackv1alpha1.Condition{}
-	} else {
-		for _, condition := range ko.Status.Conditions {
-			if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
-				resourceSyncedCondition = condition
-				break
-			}
+	for _, condition := range ko.Status.Conditions {
+		if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
+			resourceSyncedCondition = condition
+			break
 		}
 	}
 

pkg/resource/transform_job/custom_set_output.go

@@ -34,6 +34,8 @@
 )
 from e2e.replacement_values import REPLACEMENT_VALUES
 
+FAIL_UPDATE_ERROR_MESSAGE = "unable to update endpoint. check FailureReason"
+
 
 @pytest.fixture(scope="module")
 def name_suffix():
@@ -286,7 +288,7 @@ def update_endpoint_failed_test(
             endpoint_reference,
             "ACK.Terminal",
             "True",
-            "Unable to update Endpoint. Check FailureReason",
+            FAIL_UPDATE_ERROR_MESSAGE,
         )
 
         endpoint_resource = k8s.get_resource(endpoint_reference)