Update priority on a running cluster (#2369)

Author: vishalbollu

cli/cmd/cluster.go

@@ -390,7 +390,7 @@ var _clusterConfigureCmd = &cobra.Command{
 		confirmConfigureClusterConfig(configureChanges, oldClusterConfig, *newClusterConfig, _flagClusterDisallowPrompt)
 
 		out, exitCode, err := runManagerWithClusterConfig("/root/install.sh --configure", newClusterConfig, awsClient, nil, nil, []string{
-			"CORTEX_NODEGROUP_NAMES_TO_SCALE=" + strings.Join(configureChanges.NodeGroupsToScale, " "),          // NodeGroupsToScale contain the cluster config node-group names
+			"CORTEX_NODEGROUP_NAMES_TO_UPDATE=" + strings.Join(configureChanges.NodeGroupsToUpdate, " "),        // NodeGroupsToUpdate contain the cluster config node-group names
 			"CORTEX_NODEGROUP_NAMES_TO_ADD=" + strings.Join(configureChanges.NodeGroupsToAdd, " "),              // NodeGroupsToAdd contain the cluster config node-group names
 			"CORTEX_EKS_NODEGROUP_NAMES_TO_REMOVE=" + strings.Join(configureChanges.EKSNodeGroupsToRemove, " "), // EKSNodeGroupsToRemove contain the EKS node-group names
 		})

cli/cmd/lib_cluster_config.go

@@ -290,16 +290,10 @@ func confirmConfigureClusterConfig(configureChanges clusterconfig.ConfigureChang
 		fmt.Printf("￮ %s will be updated\n", fieldToUpdate)
 	}
 
-	for _, ngName := range configureChanges.NodeGroupsToScale {
+	for _, ngName := range configureChanges.NodeGroupsToUpdate {
 		ngOld := oldCc.GetNodeGroupByName(ngName)
-		ngScaled := newCc.GetNodeGroupByName(ngName)
-		if ngOld.MinInstances != ngScaled.MinInstances && ngOld.MaxInstances != ngScaled.MaxInstances {
-			fmt.Printf("￮ nodegroup %s will update %s from %d to %d and %s from %d to %d\n", ngName, clusterconfig.MinInstancesKey, ngOld.MinInstances, ngScaled.MinInstances, clusterconfig.MaxInstancesKey, ngOld.MaxInstances, ngScaled.MaxInstances)
-		} else if ngOld.MinInstances == ngScaled.MinInstances && ngOld.MaxInstances != ngScaled.MaxInstances {
-			fmt.Printf("￮ nodegroup %s will update %s from %d to %d\n", ngName, clusterconfig.MaxInstancesKey, ngOld.MaxInstances, ngScaled.MaxInstances)
-		} else if ngOld.MinInstances != ngScaled.MinInstances && ngOld.MaxInstances == ngScaled.MaxInstances {
-			fmt.Printf("￮ nodegroup %s will update %s from %d to %d\n", ngName, clusterconfig.MinInstancesKey, ngOld.MinInstances, ngScaled.MinInstances)
-		}
+		ngNew := newCc.GetNodeGroupByName(ngName)
+		fmt.Printf("￮ %s\n", ngNew.UpdatePlan(ngOld))
 	}
 
 	for _, ngName := range configureChanges.NodeGroupsToAdd {

manager/install.sh

@@ -287,15 +287,15 @@ function restart_controller_manager() {
 }
 
 function resize_nodegroups() {
-  if [ -z "$CORTEX_NODEGROUP_NAMES_TO_SCALE" ]; then
+  if [ -z "$CORTEX_NODEGROUP_NAMES_TO_UPDATE" ]; then
     return
   fi
 
   eksctl get nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 -o json > nodegroups.json
   eks_ng_len=$(cat nodegroups.json | jq -r length)
   cfg_ng_len=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups | yq -r length)
 
-  for cfg_ng_name in $CORTEX_NODEGROUP_NAMES_TO_SCALE; do
+  for cfg_ng_name in $CORTEX_NODEGROUP_NAMES_TO_UPDATE; do
     has_ng="false"
     for eks_idx in $(seq 0 $(($eks_ng_len-1))); do
       stack_ng=$(cat nodegroups.json | jq -r .[$eks_idx].Name)

pkg/types/clusterconfig/cluster_config.go

@@ -41,6 +41,7 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/sets/strset"
 	"github.com/cortexlabs/cortex/pkg/lib/slices"
 	libstr "github.com/cortexlabs/cortex/pkg/lib/strings"
+	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/lib/structs"
 	"github.com/cortexlabs/yaml"
 )
@@ -165,6 +166,27 @@ type NodeGroup struct {
 	SpotConfig               *SpotConfig `json:"spot_config" yaml:"spot_config"`
 }
 
+// compares the supported updatable fields of a nodegroup
+func (ng *NodeGroup) HasChanged(old *NodeGroup) bool {
+	return ng.MaxInstances != old.MaxInstances || ng.MinInstances != old.MinInstances || ng.Priority != old.Priority
+}
+
+func (ng *NodeGroup) UpdatePlan(old *NodeGroup) string {
+	var changes []string
+
+	if old.MinInstances != ng.MinInstances {
+		changes = append(changes, fmt.Sprintf("%s %d->%d", MinInstancesKey, old.MinInstances, ng.MinInstances))
+	}
+	if old.MaxInstances != ng.MaxInstances {
+		changes = append(changes, fmt.Sprintf("%s %d->%d", MaxInstancesKey, old.MaxInstances, ng.MaxInstances))
+	}
+	if old.Priority != ng.Priority {
+		changes = append(changes, fmt.Sprintf("%s %d->%d", PriorityKey, old.Priority, ng.Priority))
+	}
+
+	return fmt.Sprintf("nodegroup %s will be updated with the following changes: %s", ng.Name, s.StrsAnd(changes))
+}
+
 type SpotConfig struct {
 	InstanceDistribution                []string `json:"instance_distribution" yaml:"instance_distribution"`
 	OnDemandBaseCapacity                *int64   `json:"on_demand_base_capacity" yaml:"on_demand_base_capacity"`
@@ -207,13 +229,13 @@ type AccessConfig struct {
 type ConfigureChanges struct {
 	NodeGroupsToAdd       []string
 	NodeGroupsToRemove    []string
-	NodeGroupsToScale     []string
+	NodeGroupsToUpdate    []string
 	EKSNodeGroupsToRemove []string // EKS node group names of (NodeGroupsToRemove ∩ Cortex-converted EKS node groups) ∪ (Cortex-converted EKS node groups - the new cluster config's nodegroups)
 	FieldsToUpdate        []string
 }
 
 func (c *ConfigureChanges) HasChanges() bool {
-	return len(c.NodeGroupsToAdd)+len(c.NodeGroupsToRemove)+len(c.NodeGroupsToScale)+len(c.EKSNodeGroupsToRemove)+len(c.FieldsToUpdate) != 0
+	return len(c.NodeGroupsToAdd)+len(c.NodeGroupsToRemove)+len(c.NodeGroupsToUpdate)+len(c.EKSNodeGroupsToRemove)+len(c.FieldsToUpdate) != 0
 }
 
 // GetGhostEKSNodeGroups returns the set difference between EKSNodeGroupsToRemove and the EKS-converted NodeGroupsToRemove
@@ -1087,8 +1109,10 @@ func (cc *Config) validateSharedNodeGroupsDiff(oldConfig Config) error {
 
 		newNgCopy.MinInstances = 0
 		newNgCopy.MaxInstances = 0
+		newNgCopy.Priority = 0
 		oldNgCopy.MinInstances = 0
 		oldNgCopy.MaxInstances = 0
+		oldNgCopy.Priority = 0
 
 		newHash, err := newNgCopy.Hash()
 		if err != nil {
@@ -1200,17 +1224,17 @@ func (cc *Config) ValidateOnConfigure(awsClient *aws.Client, k8sClient *k8s.Clie
 	}
 
 	sharedNgsFromNewConfig, sharedNgsFromOldConfig := cc.getCommonNodeGroups(oldConfig)
-	ngNamesToBeScaled := []*NodeGroup{}
+	ngsToBeUpdated := []*NodeGroup{}
 	for i := range sharedNgsFromNewConfig {
-		if sharedNgsFromNewConfig[i].MinInstances != sharedNgsFromOldConfig[i].MinInstances || sharedNgsFromNewConfig[i].MaxInstances != sharedNgsFromOldConfig[i].MaxInstances {
-			ngNamesToBeScaled = append(ngNamesToBeScaled, sharedNgsFromNewConfig[i])
+		if sharedNgsFromNewConfig[i].HasChanged(sharedNgsFromOldConfig[i]) {
+			ngsToBeUpdated = append(ngsToBeUpdated, sharedNgsFromNewConfig[i])
 		}
 	}
 
 	return ConfigureChanges{
 		NodeGroupsToAdd:       GetNodeGroupNames(ngsToBeAdded),
 		NodeGroupsToRemove:    GetNodeGroupNames(ngsToBeRemoved),
-		NodeGroupsToScale:     GetNodeGroupNames(ngNamesToBeScaled),
+		NodeGroupsToUpdate:    GetNodeGroupNames(ngsToBeUpdated),
 		EKSNodeGroupsToRemove: getStaleEksNodeGroups(cc.ClusterName, eksNodeGroupStacks, cc.NodeGroups, ngsToBeRemoved),
 		FieldsToUpdate:        fieldsToUpdate,
 	}, nil

pkg/types/clusterconfig/config_key.go

@@ -29,6 +29,7 @@ const (
 	AcceleratorsPerInstanceKey             = "accelerators_per_instance"
 	MinInstancesKey                        = "min_instances"
 	MaxInstancesKey                        = "max_instances"
+	PriorityKey                            = "priority"
 	SpotKey                                = "spot"
 	SpotConfigKey                          = "spot_config"
 	InstanceDistributionKey                = "instance_distribution"