Support GCP network configuration (#1752)

Author: deliahu

cli/cmd/cluster_gcp.go

@@ -434,53 +434,62 @@ func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Cli
 	gkeClusterParent := fmt.Sprintf("projects/%s/locations/%s", *clusterConfig.Project, *clusterConfig.Zone)
 	gkeClusterName := fmt.Sprintf("%s/clusters/%s", gkeClusterParent, clusterConfig.ClusterName)
 
-	_, err := gcpClient.CreateCluster(&containerpb.CreateClusterRequest{
-		Parent: gkeClusterParent,
-		Cluster: &containerpb.Cluster{
-			Name:                  clusterConfig.ClusterName,
-			InitialClusterVersion: "1.17",
-			NodePools: []*containerpb.NodePool{
-				{
-					Name: "ng-cortex-operator",
-					Config: &containerpb.NodeConfig{
-						MachineType: "n1-standard-2",
-						OauthScopes: []string{
-							"https://www.googleapis.com/auth/compute",
-							"https://www.googleapis.com/auth/devstorage.read_only",
-						},
-						ServiceAccount: gcpClient.ClientEmail,
+	gkeClusterConfig := containerpb.Cluster{
+		Name:                  clusterConfig.ClusterName,
+		InitialClusterVersion: "1.17",
+		NodePools: []*containerpb.NodePool{
+			{
+				Name: "ng-cortex-operator",
+				Config: &containerpb.NodeConfig{
+					MachineType: "n1-standard-2",
+					OauthScopes: []string{
+						"https://www.googleapis.com/auth/compute",
+						"https://www.googleapis.com/auth/devstorage.read_only",
 					},
-					InitialNodeCount: 1,
+					ServiceAccount: gcpClient.ClientEmail,
 				},
-				{
-					Name: "ng-cortex-worker-on-demand",
-					Config: &containerpb.NodeConfig{
-						MachineType: *clusterConfig.InstanceType,
-						Labels:      nodeLabels,
-						Taints: []*containerpb.NodeTaint{
-							{
-								Key:    "workload",
-								Value:  "true",
-								Effect: containerpb.NodeTaint_NO_SCHEDULE,
-							},
-						},
-						Accelerators: accelerators,
-						OauthScopes: []string{
-							"https://www.googleapis.com/auth/compute",
-							"https://www.googleapis.com/auth/devstorage.read_only",
+				InitialNodeCount: 1,
+			},
+			{
+				Name: "ng-cortex-worker-on-demand",
+				Config: &containerpb.NodeConfig{
+					MachineType: *clusterConfig.InstanceType,
+					Labels:      nodeLabels,
+					Taints: []*containerpb.NodeTaint{
+						{
+							Key:    "workload",
+							Value:  "true",
+							Effect: containerpb.NodeTaint_NO_SCHEDULE,
 						},
-						ServiceAccount: gcpClient.ClientEmail,
 					},
-					Autoscaling: &containerpb.NodePoolAutoscaling{
-						Enabled:      true,
-						MinNodeCount: int32(*clusterConfig.MinInstances),
-						MaxNodeCount: int32(*clusterConfig.MaxInstances),
+					Accelerators: accelerators,
+					OauthScopes: []string{
+						"https://www.googleapis.com/auth/compute",
+						"https://www.googleapis.com/auth/devstorage.read_only",
 					},
-					InitialNodeCount: int32(*clusterConfig.MinInstances),
+					ServiceAccount: gcpClient.ClientEmail,
+				},
+				Autoscaling: &containerpb.NodePoolAutoscaling{
+					Enabled:      true,
+					MinNodeCount: int32(*clusterConfig.MinInstances),
+					MaxNodeCount: int32(*clusterConfig.MaxInstances),
 				},
+				InitialNodeCount: int32(*clusterConfig.MinInstances),
 			},
-			Locations: []string{*clusterConfig.Zone},
 		},
+		Locations: []string{*clusterConfig.Zone},
+	}
+
+	if clusterConfig.Network != nil {
+		gkeClusterConfig.Network = *clusterConfig.Network
+	}
+	if clusterConfig.Subnet != nil {
+		gkeClusterConfig.Subnetwork = *clusterConfig.Subnet
+	}
+
+	_, err := gcpClient.CreateCluster(&containerpb.CreateClusterRequest{
+		Parent:  gkeClusterParent,
+		Cluster: &gkeClusterConfig,
 	})
 	if err != nil {
 		return err

docs/clusters/gcp/install.md

@@ -31,14 +31,20 @@ zone: us-central1-a
 # instance type
 instance_type: n1-standard-2
 
-# GPU to attach to your instance (optional)
-accelerator_type: nvidia-tesla-t4
-
 # minimum number of instances
 min_instances: 1
 
 # maximum number of instances
 max_instances: 5
+
+# GPU to attach to your instance (optional)
+# accelerator_type: nvidia-tesla-t4
+
+# the name of the network in which to create your cluster
+# network: default
+
+# the name of the subnetwork in which to create your cluster
+# subnet: default
 ```
 
 The docker images used by the Cortex cluster can also be overridden, although this is not common. They can be configured by adding any of these keys to your cluster configuration file (default values are shown):

pkg/types/clusterconfig/cluster_config_gcp.go

@@ -38,6 +38,8 @@ type GCPConfig struct {
 	Zone             *string            `json:"zone" yaml:"zone"`
 	InstanceType     *string            `json:"instance_type" yaml:"instance_type"`
 	AcceleratorType  *string            `json:"accelerator_type" yaml:"accelerator_type"`
+	Network          *string            `json:"network" yaml:"network"`
+	Subnet           *string            `json:"subnet" yaml:"subnet"`
 	MinInstances     *int64             `json:"min_instances" yaml:"min_instances"`
 	MaxInstances     *int64             `json:"max_instances" yaml:"max_instances"`
 	ClusterName      string             `json:"cluster_name" yaml:"cluster_name"`
@@ -133,6 +135,18 @@ var UserGCPValidation = &cr.StructValidation{
 				AllowExplicitNull: true,
 			},
 		},
+		{
+			StructField: "Network",
+			StringPtrValidation: &cr.StringPtrValidation{
+				AllowExplicitNull: true,
+			},
+		},
+		{
+			StructField: "Subnet",
+			StringPtrValidation: &cr.StringPtrValidation{
+				AllowExplicitNull: true,
+			},
+		},
 		{
 			StructField: "MinInstances",
 			Int64PtrValidation: &cr.Int64PtrValidation{
@@ -473,6 +487,12 @@ func (cc *GCPConfig) UserTable() table.KeyValuePairs {
 	if cc.AcceleratorType != nil {
 		items.Add(AcceleratorTypeUserKey, *cc.AcceleratorType)
 	}
+	if cc.Network != nil {
+		items.Add(NetworkUserKey, *cc.Network)
+	}
+	if cc.Subnet != nil {
+		items.Add(SubnetUserKey, *cc.Subnet)
+	}
 	items.Add(TelemetryUserKey, cc.Telemetry)
 	items.Add(ImageOperatorUserKey, cc.ImageOperator)
 	items.Add(ImageManagerUserKey, cc.ImageManager)
@@ -501,6 +521,12 @@ func (cc *GCPConfig) TelemetryEvent() map[string]interface{} {
 		event["accelerator_type._is_defined"] = true
 		event["accelerator_type"] = *cc.AcceleratorType
 	}
+	if cc.Network != nil {
+		event["network._is_defined"] = true
+	}
+	if cc.Subnet != nil {
+		event["subnet._is_defined"] = true
+	}
 	if cc.MinInstances != nil {
 		event["min_instances._is_defined"] = true
 		event["min_instances"] = *cc.MinInstances

pkg/types/clusterconfig/config_key.go

@@ -20,6 +20,8 @@ const (
 	ProviderKey                            = "provider"
 	InstanceTypeKey                        = "instance_type"
 	AcceleratorTypeKey                     = "accelerator_type"
+	NetworkKey                             = "network"
+	SubnetKey                              = "subnet"
 	MinInstancesKey                        = "min_instances"
 	MaxInstancesKey                        = "max_instances"
 	TagsKey                                = "tags"
@@ -76,6 +78,8 @@ const (
 	SpotUserKey                                = "use spot instances"
 	InstanceTypeUserKey                        = "instance type"
 	AcceleratorTypeUserKey                     = "accelerator type"
+	NetworkUserKey                             = "network"
+	SubnetUserKey                              = "subnet"
 	MinInstancesUserKey                        = "min instances"
 	MaxInstancesUserKey                        = "max instances"
 	TagsUserKey                                = "tags"