Miscellaneous UI improvements (#1065)

Author: deliahu

cli/cmd/lib_cluster_config.go

@@ -74,7 +74,7 @@ func readCachedClusterConfigFile(clusterConfig *clusterconfig.Config, filePath s
 func readUserClusterConfigFile(clusterConfig *clusterconfig.Config) error {
 	errs := cr.ParseYAMLFile(clusterConfig, clusterconfig.UserValidation, _flagClusterConfig)
 	if errors.HasError(errs) {
-		return errors.FirstError(errs...)
+		return errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://www.cortex.dev/v/%s/cluster-management/config", consts.CortexVersionMinor))
 	}
 
 	return nil
@@ -89,7 +89,7 @@ func getClusterAccessConfig(disallowPrompt bool) (*clusterconfig.AccessConfig, e
 	if _flagClusterConfig != "" {
 		errs := cr.ParseYAMLFile(accessConfig, clusterconfig.AccessValidation, _flagClusterConfig)
 		if errors.HasError(errs) {
-			return nil, errors.FirstError(errs...)
+			return nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found here: https://www.cortex.dev/v/%s/cluster-management/config", consts.CortexVersionMinor))
 		}
 	}
 
@@ -157,7 +157,7 @@ func getInstallClusterConfig(awsCreds AWSCredentials, envName string, disallowPr
 	}
 	promptIfNotAdmin(awsClient, disallowPrompt)
 
-	err = clusterconfig.InstallPrompt(clusterConfig, awsClient, disallowPrompt)
+	err = clusterconfig.InstallPrompt(clusterConfig, disallowPrompt)
 	if err != nil {
 		return nil, err
 	}

docs/cluster-management/spot-instances.md

@@ -12,7 +12,7 @@ spot: false
 
 spot_config:
   # additional instances with identical or better specs than the primary instance type (defaults to only the primary instance)
-  instance_distribution: [similar_instance_type_1, similar_instance_type_2]
+  instance_distribution: # [similar_instance_type_1, similar_instance_type_2]
 
   # minimum number of on demand instances (default: 0)
   on_demand_base_capacity: 0
@@ -22,7 +22,7 @@ spot_config:
   on_demand_percentage_above_base_capacity: 0
 
   # max price for spot instances (default: the on-demand price of the primary instance type)
-  max_price: <float>
+  max_price: # <float>
 
   # number of spot instance pools across which to allocate spot instances [1, 20] (default: number of instances in instance distribution)
   instance_pools: 3

pkg/lib/aws/ec2.go

@@ -128,7 +128,7 @@ func (c *Client) ListRegions() (strset.Set, strset.Set, error) {
 	return allRegions, enabledRegions, nil
 }
 
-func (c *Client) ListAvailabilityZones() (strset.Set, error) {
+func (c *Client) ListAvailabilityZonesInRegion() (strset.Set, error) {
 	input := &ec2.DescribeAvailabilityZonesInput{
 		Filters: []*ec2.Filter{
 			{

pkg/lib/aws/s3.go

@@ -553,7 +553,7 @@ func (c *Client) ListS3Prefix(bucket string, prefix string, includeDirObjects bo
 	})
 
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, S3Path(bucket, prefix))
 	}
 
 	return allObjects, nil

pkg/lib/files/files.go

@@ -822,10 +822,23 @@ func HashFile(path string, paths ...string) (string, error) {
 			return "", err
 		}
 
+		// Skip directories
+		fileInfo, err := f.Stat()
+		if err != nil {
+			f.Close()
+			return "", errors.WithStack(err)
+		}
+		if fileInfo.IsDir() {
+			f.Close()
+			continue
+		}
+
 		if _, err := io.Copy(md5Hash, f); err != nil {
 			f.Close()
 			return "", errors.Wrap(err, path)
 		}
+
+		io.WriteString(md5Hash, path)
 		f.Close()
 	}
 
@@ -865,6 +878,7 @@ func HashDirectory(dir string, ignoreFns ...IgnoreFn) (string, error) {
 			return errors.Wrap(err, path)
 		}
 
+		io.WriteString(md5Hash, path)
 		return nil
 	})
 

pkg/types/clusterconfig/availability_zones.go

@@ -42,7 +42,7 @@ func (cc *Config) setDefaultAvailabilityZones(awsClient *aws.Client, extraInstan
 	zones, err := awsClient.ListSupportedAvailabilityZones(*cc.InstanceType, extraInstances...)
 	if err != nil {
 		// Try again without checking instance types
-		zones, err = awsClient.ListAvailabilityZones()
+		zones, err = awsClient.ListAvailabilityZonesInRegion()
 		if err != nil {
 			return nil // Let eksctl choose the availability zones
 		}
@@ -67,14 +67,14 @@ func (cc *Config) setDefaultAvailabilityZones(awsClient *aws.Client, extraInstan
 }
 
 func (cc *Config) validateUserAvailabilityZones(awsClient *aws.Client, extraInstances ...string) error {
-	allZones, err := awsClient.ListAvailabilityZones()
+	allZones, err := awsClient.ListAvailabilityZonesInRegion()
 	if err != nil {
 		return nil // Skip validation
 	}
 
 	for _, userZone := range cc.AvailabilityZones {
 		if !allZones.Has(userZone) {
-			return ErrorInvalidAvailabilityZone(userZone, allZones)
+			return ErrorInvalidAvailabilityZone(userZone, allZones, awsClient.Region)
 		}
 	}
 

pkg/types/clusterconfig/clusterconfig.go

@@ -498,9 +498,28 @@ func (cc *Config) Validate(awsClient *aws.Client) error {
 		return ErrorNATRequiredWithPrivateSubnetVisibility()
 	}
 
-	bucketRegion, _ := aws.GetBucketRegion(cc.Bucket)
-	if bucketRegion != "" && bucketRegion != *cc.Region { // if the bucket didn't exist, we will create it in the correct region, so there is no error
-		return ErrorS3RegionDiffersFromCluster(cc.Bucket, bucketRegion, *cc.Region)
+	if cc.Bucket == "" {
+		accountID, _, err := awsClient.GetCachedAccountID()
+		if err != nil {
+			return err
+		}
+
+		bucketID := hash.String(accountID + *cc.Region)[:10]
+
+		defaultBucket := cc.ClusterName + "-" + bucketID
+		if len(defaultBucket) > 63 {
+			defaultBucket = defaultBucket[:63]
+		}
+		if strings.HasSuffix(defaultBucket, "-") {
+			defaultBucket = defaultBucket[:len(defaultBucket)-1]
+		}
+
+		cc.Bucket = defaultBucket
+	} else {
+		bucketRegion, _ := aws.GetBucketRegion(cc.Bucket)
+		if bucketRegion != "" && bucketRegion != *cc.Region { // if the bucket didn't exist, we will create it in the correct region, so there is no error
+			return ErrorS3RegionDiffersFromCluster(cc.Bucket, bucketRegion, *cc.Region)
+		}
 	}
 
 	primaryInstanceType := *cc.InstanceType
@@ -740,26 +759,10 @@ func RegionPrompt(clusterConfig *Config, disallowPrompt bool) error {
 	return nil
 }
 
-func InstallPrompt(clusterConfig *Config, awsClient *aws.Client, disallowPrompt bool) error {
+func InstallPrompt(clusterConfig *Config, disallowPrompt bool) error {
 	defaults := applyPromptDefaults(*clusterConfig)
-	accountID, _, err := awsClient.GetCachedAccountID()
-	if err != nil {
-		return err
-	}
-	bucketID := hash.String(accountID + *clusterConfig.Region)[:10]
-
-	defaultBucket := clusterConfig.ClusterName + "-" + bucketID
-	if len(defaultBucket) > 63 {
-		defaultBucket = defaultBucket[:63]
-	}
-	if strings.HasSuffix(defaultBucket, "-") {
-		defaultBucket = defaultBucket[:len(defaultBucket)-1]
-	}
 
 	if disallowPrompt {
-		if clusterConfig.Bucket == "" {
-			clusterConfig.Bucket = defaultBucket
-		}
 		if clusterConfig.InstanceType == nil {
 			clusterConfig.InstanceType = defaults.InstanceType
 		}
@@ -775,18 +778,6 @@ func InstallPrompt(clusterConfig *Config, awsClient *aws.Client, disallowPrompt
 	remainingPrompts := &cr.PromptValidation{
 		SkipNonEmptyFields: true,
 		PromptItemValidations: []*cr.PromptItemValidation{
-			{
-				StructField: "Bucket",
-				PromptOpts: &prompt.Options{
-					Prompt: BucketUserKey,
-				},
-				StringValidation: &cr.StringValidation{
-					Default:   defaultBucket,
-					MinLength: 3,
-					MaxLength: 63,
-					Validator: validateBucketName,
-				},
-			},
 			{
 				StructField: "InstanceType",
 				PromptOpts: &prompt.Options{
@@ -823,7 +814,7 @@ func InstallPrompt(clusterConfig *Config, awsClient *aws.Client, disallowPrompt
 		},
 	}
 
-	err = cr.ReadPrompt(clusterConfig, remainingPrompts)
+	err := cr.ReadPrompt(clusterConfig, remainingPrompts)
 	if err != nil {
 		return err
 	}

pkg/types/clusterconfig/errors.go

@@ -145,10 +145,10 @@ func ErrorConfigCannotBeChangedOnUpdate(configKey string, prevVal interface{}) e
 	})
 }
 
-func ErrorInvalidAvailabilityZone(userZone string, allZones strset.Set) error {
+func ErrorInvalidAvailabilityZone(userZone string, allZones strset.Set, region string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrInvalidAvailabilityZone,
-		Message: fmt.Sprintf("%s is not an availability zone; please choose from the following availability zones: %s", s.UserStr(userZone), s.UserStrsOr(allZones.SliceSorted())),
+		Message: fmt.Sprintf("%s is not an availability zone in %s; please choose from the following availability zones: %s", s.UserStr(userZone), region, s.UserStrsOr(allZones.SliceSorted())),
 	})
 }
 

pkg/types/spec/errors.go

@@ -39,7 +39,6 @@ const (
 	ErrInitReplicasLessThanMin              = "spec.init_replicas_less_than_min"
 	ErrInvalidSurgeOrUnavailable            = "spec.invalid_surge_or_unavailable"
 	ErrSurgeAndUnavailableBothZero          = "spec.surge_and_unavailable_both_zero"
-	ErrImplDoesNotExist                     = "spec.impl_does_not_exist"
 	ErrFileNotFound                         = "spec.file_not_found"
 	ErrDirIsEmpty                           = "spec.dir_is_empty"
 	ErrS3FileNotFound                       = "spec.s3_file_not_found"
@@ -166,13 +165,6 @@ func ErrorSurgeAndUnavailableBothZero() error {
 	})
 }
 
-func ErrorImplDoesNotExist(path string) error {
-	return errors.WithStack(&errors.Error{
-		Kind:    ErrImplDoesNotExist,
-		Message: fmt.Sprintf("%s: implementation file does not exist", path),
-	})
-}
-
 func ErrorFileNotFound(path string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrFileNotFound,

pkg/types/spec/validations.go

@@ -17,6 +17,7 @@ limitations under the License.
 package spec
 
 import (
+	"fmt"
 	"math"
 	"path/filepath"
 	"strconv"
@@ -158,7 +159,8 @@ func monitoringValidation() *cr.StructFieldValidation {
 	return &cr.StructFieldValidation{
 		StructField: "Monitoring",
 		StructValidation: &cr.StructValidation{
-			DefaultNil: true,
+			DefaultNil:        true,
+			AllowExplicitNull: true,
 			StructFieldValidations: []*cr.StructFieldValidation{
 				{
 					StructField:         "Key",
@@ -410,7 +412,8 @@ func ExtractAPIConfigs(configBytes []byte, provider types.ProviderType, projectF
 		errs := cr.Struct(&api, data, apiValidation(provider))
 		if errors.HasError(errs) {
 			name, _ := data[userconfig.NameKey].(string)
-			return nil, errors.Wrap(errors.FirstError(errs...), userconfig.IdentifyAPI(filePath, name, i))
+			err = errors.Wrap(errors.FirstError(errs...), userconfig.IdentifyAPI(filePath, name, i))
+			return nil, errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found here: https://www.cortex.dev/v/%s/deployments/api-configuration", consts.CortexVersionMinor))
 		}
 		api.Index = i
 		api.FilePath = filePath
@@ -483,7 +486,7 @@ func validatePredictor(predictor *userconfig.Predictor, projectFiles ProjectFile
 
 	if _, err := projectFiles.GetFile(predictor.Path); err != nil {
 		if errors.GetKind(err) == files.ErrFileDoesNotExist {
-			return errors.Wrap(ErrorImplDoesNotExist(predictor.Path), userconfig.PathKey)
+			return errors.Wrap(files.ErrorFileDoesNotExist(predictor.Path), userconfig.PathKey)
 		}
 		return errors.Wrap(err, userconfig.PathKey)
 	}
@@ -766,7 +769,7 @@ func validatePythonPath(pythonPath string, projectFiles ProjectFiles) error {
 		}
 	}
 	if !validPythonPath {
-		return ErrorImplDoesNotExist(pythonPath)
+		return files.ErrorFileDoesNotExist(pythonPath)
 	}
 	return nil
 }