Add retries for resource tagging (#1188)

(cherry picked from commit 8b5d3a3)

Author: deliahu

cli/cmd/cluster.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
 	"github.com/cortexlabs/cortex/cli/cluster"
 	"github.com/cortexlabs/cortex/cli/types/cliconfig"
 	"github.com/cortexlabs/cortex/pkg/consts"
@@ -138,20 +139,12 @@ var _upCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		err = createBucketIfNotFound(awsClient, clusterConfig.Bucket)
-		if err != nil {
-			exit.Error(err)
-		}
-		err = awsClient.TagBucket(clusterConfig.Bucket, clusterConfig.Tags)
+		err = createBucketIfNotFound(awsClient, clusterConfig.Bucket, clusterConfig.Tags)
 		if err != nil {
 			exit.Error(err)
 		}
 
-		err = createLogGroupIfNotFound(awsClient, clusterConfig.LogGroup)
-		if err != nil {
-			exit.Error(err)
-		}
-		err = awsClient.TagLogGroup(clusterConfig.LogGroup, clusterConfig.Tags)
+		err = createLogGroupIfNotFound(awsClient, clusterConfig.LogGroup, clusterConfig.Tags)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -725,7 +718,7 @@ func getCloudFormationURLWithAccessConfig(accessConfig *clusterconfig.AccessConf
 	return getCloudFormationURL(*accessConfig.ClusterName, *accessConfig.Region)
 }
 
-func createBucketIfNotFound(awsClient *aws.Client, bucket string) error {
+func createBucketIfNotFound(awsClient *aws.Client, bucket string, tags map[string]string) error {
 	bucketFound, err := awsClient.DoesBucketExist(bucket)
 	if err != nil {
 		return err
@@ -737,14 +730,28 @@ func createBucketIfNotFound(awsClient *aws.Client, bucket string) error {
 			fmt.Print("\n\n")
 			return err
 		}
-		fmt.Println(" ✓")
 	} else {
-		fmt.Println("￮ using existing s3 bucket:", bucket, "✓")
+		fmt.Print("￮ using existing s3 bucket:", bucket)
 	}
-	return nil
+
+	// retry since it's possible that it takes some time for the new bucket to be registered by AWS
+	for i := 0; i < 10; i++ {
+		err = awsClient.TagBucket(bucket, tags)
+		if err == nil {
+			fmt.Println(" ✓")
+			return nil
+		}
+		if !aws.IsNoSuchBucketErr(err) {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	fmt.Print("\n\n")
+	return err
 }
 
-func createLogGroupIfNotFound(awsClient *aws.Client, logGroup string) error {
+func createLogGroupIfNotFound(awsClient *aws.Client, logGroup string, tags map[string]string) error {
 	logGroupFound, err := awsClient.DoesLogGroupExist(logGroup)
 	if err != nil {
 		return err
@@ -756,12 +763,25 @@ func createLogGroupIfNotFound(awsClient *aws.Client, logGroup string) error {
 			fmt.Print("\n\n")
 			return err
 		}
-		fmt.Println(" ✓")
 	} else {
-		fmt.Println("￮ using existing cloudwatch log group:", logGroup, "✓")
+		fmt.Print("￮ using existing cloudwatch log group:", logGroup)
 	}
 
-	return nil
+	// retry since it's possible that it takes some time for the new log group to be registered by AWS
+	for i := 0; i < 10; i++ {
+		err = awsClient.TagLogGroup(logGroup, tags)
+		if err == nil {
+			fmt.Println(" ✓")
+			return nil
+		}
+		if !aws.IsErrCode(err, cloudwatchlogs.ErrCodeResourceNotFoundException) {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	fmt.Print("\n\n")
+	return err
 }
 
 // createOrClearDashboard creates a new dashboard (or clears an existing one if it already exists)

pkg/lib/aws/errors.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 )
@@ -43,15 +44,11 @@ func IsNotFoundErr(err error) bool {
 }
 
 func IsNoSuchKeyErr(err error) bool {
-	return IsErrCode(err, "NoSuchKey")
+	return IsErrCode(err, s3.ErrCodeNoSuchKey)
 }
 
 func IsNoSuchBucketErr(err error) bool {
-	return IsErrCode(err, "NoSuchBucket")
-}
-
-func IsForbiddenErr(err error) bool {
-	return IsErrCode(err, "Forbidden")
+	return IsErrCode(err, s3.ErrCodeNoSuchBucket)
 }
 
 func IsGenericNotFoundErr(err error) bool {