init

Author: phuhung273

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/aws/aws-sdk-go v1.44.327
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/container-storage-interface/spec v1.7.0
 	github.com/golang/mock v1.6.0
 	github.com/kubernetes-csi/csi-test/v5 v5.0.0
@@ -29,7 +30,6 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.4.0 // indirect

go.sum

@@ -61,6 +61,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
 github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

pkg/driver/controller.go

@@ -19,6 +19,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/pkg/cloud"
 	"github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/pkg/driver/internal"
@@ -27,9 +33,6 @@ import (
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/timestamppb"
 	"k8s.io/klog/v2"
-	"os"
-	"strconv"
-	"strings"
 )
 
 var (
@@ -95,6 +98,7 @@ type controllerService struct {
 	cloud         cloud.Cloud
 	inFlight      *internal.InFlight
 	driverOptions *DriverOptions
+	backoff       backoff.BackOff
 }
 
 // newControllerService creates a new controller service
@@ -122,6 +126,7 @@ func newControllerService(driverOptions *DriverOptions) controllerService {
 		cloud:         cloudSrv,
 		inFlight:      internal.NewInFlight(),
 		driverOptions: driverOptions,
+		backoff:       backoff.NewExponentialBackOff(), // Default spec: https://pkg.go.dev/github.com/cenkalti/backoff/v4#pkg-constants
 	}
 }
 
@@ -355,6 +360,30 @@ func (d *controllerService) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 		return nil, status.Errorf(codes.Internal, "Could not delete volume ID %q: %v", volumeID, err)
 	}
 
+	// Wait until volume not found
+	describeRetry := func() error {
+		if splitVolumeId[0] == cloud.FilesystemPrefix {
+			_, err = d.cloud.DescribeFileSystem(ctx, volumeID)
+		}
+		if splitVolumeId[0] == cloud.VolumePrefix {
+			_, err = d.cloud.DescribeVolume(ctx, volumeID)
+		}
+
+		if err != nil {
+			if err == cloud.ErrNotFound {
+				return nil
+			}
+			return status.Error(codes.Internal, err.Error())
+		}
+		return status.Errorf(codes.Internal, "Volume ID %q still exists", volumeID)
+	}
+
+	err = backoff.RetryNotify(describeRetry, d.backoff, notifyRetryError)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Could not delete volume ID %q: %v", volumeID, err)
+	}
+
+	klog.V(4).InfoS("DeleteVolume: volume not found, returning with success", "volumeId", volumeID)
 	return &csi.DeleteVolumeResponse{}, nil
 }
 
@@ -785,3 +814,7 @@ func (d *controllerService) appendSnapshotARN(ctx context.Context, parameters ma
 
 	return nil
 }
+
+func notifyRetryError(err error, t time.Duration) {
+	klog.ErrorS(err, fmt.Sprintf("Retrying in %f seconds", t.Seconds()))
+}