add support for create vpc-cni network cluster

Author: howardxiao(肖华源)

examples/tencentcloud-tke/main.tf

@@ -41,6 +41,59 @@ resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
   }
 }
 
+#examples for MANAGED_CLUSTER VPC-CNI network type cluster with customized master params
+resource "tencentcloud_kubernetes_cluster" "managed_vpc_cni_cluster" {
+  cluster_version         = "1.14.3"
+  vpc_id                  = var.vpc
+  cluster_max_pod_num     = 32
+  cluster_name            = "testvpccni"
+  cluster_desc            = "test vpc-cni cluster desc"
+  cluster_max_service_num = 32
+  service_cidr            = "192.168.128.0/24"
+  eni_subnet_ids          = ["subnet-hmmlszs7", "subnet-4o0v4e7j"]
+  claim_expired_seconds   = 300
+  network_type            = "VPC-CNI"
+  is_non_static_ip_mode   = true
+  cluster_extra_args {
+    kube_apiserver          = ["max-requests-inflight=450"]
+    kube_controller_manager = ["kube-api-burst=500", "kube-api-qps=200"]
+    kube_scheduler          = ["kube-api-burst=500", "kube-api-qps=200"]
+  }
+
+  worker_config {
+    count                      = 2
+    availability_zone          = var.availability_zone
+    instance_type              = var.default_instance_type
+    system_disk_type           = "CLOUD_SSD"
+    system_disk_size           = 60
+    internet_charge_type       = "TRAFFIC_POSTPAID_BY_HOUR"
+    internet_max_bandwidth_out = 100
+    public_ip_assigned         = true
+    subnet_id                  = var.subnet
+
+    data_disk {
+      disk_type = "CLOUD_PREMIUM"
+      disk_size = 50
+    }
+
+    enhanced_security_service = false
+    enhanced_monitor_service  = false
+    user_data                 = "dGVzdA=="
+    password                  = "ZZXXccvv1212"
+  }
+
+  cluster_deploy_type = "MANAGED_CLUSTER"
+
+  tags = {
+    "test" = "test"
+  }
+
+  labels = {
+    "test1" = "test1",
+    "test2" = "test2",
+  }
+}
+
 #examples for INDEPENDENT_CLUSTER  cluster
 resource "tencentcloud_kubernetes_cluster" "independing_cluster" {
   vpc_id                  = var.vpc

tencentcloud/data_source_tc_kubernetes_clusters.go

@@ -225,12 +225,22 @@ LOOP:
 		infoMap["cluster_deploy_type"] = info.DeployType
 		infoMap["cluster_version"] = info.ClusterVersion
 		infoMap["cluster_ipvs"] = info.Ipvs
+		infoMap["cluster_as_enabled"] = info.AsEnabled
+		infoMap["node_name_type"] = info.NodeNameType
+		infoMap["cluster_extra_args"] = info.ExtraArgs
+		infoMap["network_type"] = info.NetworkType
+		infoMap["is_non_static_ip_mode"] = info.IsNonStaticIpMode
+		infoMap["deletion_protection"] = info.DeletionProtection
+		infoMap["kube_proxy_mode"] = info.KubeProxyMode
 		infoMap["vpc_id"] = info.VpcId
 		infoMap["project_id"] = info.ProjectId
 		infoMap["cluster_cidr"] = info.ClusterCidr
 		infoMap["ignore_cluster_cidr_conflict"] = info.IgnoreClusterCidrConflict
-		infoMap["cluster_max_pod_num"] = info.MaxClusterServiceNum
+		infoMap["cluster_max_pod_num"] = info.MaxNodePodNum
 		infoMap["cluster_max_service_num"] = info.MaxClusterServiceNum
+		infoMap["service_cidr"] = info.ServiceCIDR
+		infoMap["eni_subnet_ids"] = info.EniSubnetIds
+		infoMap["claim_expired_seconds"] = info.ClaimExpiredSeconds
 		infoMap["cluster_node_num"] = info.ClusterNodeNum
 		infoMap["tags"] = info.Tags
 

tencentcloud/extension_tke.go

@@ -61,3 +61,23 @@ const (
 	TkeInternetStatusDeletedFailed = "DeletedFailed"
 	TkeInternetStatusNotfound      = "NotFound"
 )
+
+const (
+	TKE_CLUSTER_NETWORK_TYPE_GR      = "GR"
+	TKE_CLUSTER_NETWORK_TYPE_VPC_CNI = "VPC-CNI"
+)
+
+var TKE_CLUSTER_NETWORK_TYPE = []string{TKE_CLUSTER_NETWORK_TYPE_GR, TKE_CLUSTER_NETWORK_TYPE_VPC_CNI}
+
+const (
+	TKE_CLUSTER_NODE_NAME_TYPE_LAN_IP   = "lan-ip"
+	TKE_CLUSTER_NODE_NAME_TYPE_HOSTNAME = "hostname"
+)
+
+var TKE_CLUSTER_NODE_NAME_TYPE = []string{TKE_CLUSTER_NODE_NAME_TYPE_LAN_IP, TKE_CLUSTER_NODE_NAME_TYPE_HOSTNAME}
+
+const (
+	TKE_CLUSTER_KUBE_PROXY_MODE_BPF = "kube-proxy-bpf"
+)
+
+var TKE_CLUSTER_KUBE_PROXY_MODE = []string{TKE_CLUSTER_KUBE_PROXY_MODE_BPF}

tencentcloud/resource_tc_kubernetes_cluster.go

@@ -414,6 +414,82 @@ func resourceTencentCloudTkeCluster() *schema.Resource {
 			Default:     true,
 			Description: "Indicates whether ipvs is enabled. Default is true.",
 		},
+		"cluster_as_enabled": {
+			Type:        schema.TypeBool,
+			ForceNew:    true,
+			Optional:    true,
+			Default:     false,
+			Description: "Indicates whether to enable cluster node auto scaler.",
+		},
+		"cluster_extra_args": {
+			Type:     schema.TypeList,
+			ForceNew: true,
+			Optional: true,
+			MaxItems: 1,
+			Elem: &schema.Resource{
+				Schema: map[string]*schema.Schema{
+					"kube_apiserver": {
+						Type:        schema.TypeList,
+						ForceNew:    true,
+						Optional:    true,
+						Elem:        &schema.Schema{Type: schema.TypeString},
+						Description: "The customized parameters for kube-apiserver.",
+					},
+					"kube_controller_manager": {
+						Type:        schema.TypeList,
+						ForceNew:    true,
+						Optional:    true,
+						Elem:        &schema.Schema{Type: schema.TypeString},
+						Description: "The customized parameters for kube-controller-manager.",
+					},
+					"kube_scheduler": {
+						Type:        schema.TypeList,
+						ForceNew:    true,
+						Optional:    true,
+						Elem:        &schema.Schema{Type: schema.TypeString},
+						Description: "The customized parameters for kube-scheduler.",
+					},
+				},
+			},
+			Description: "Customized parameters for master component,such as kube-apiserver, kube-controller-manager, kube-scheduler.",
+		},
+		"node_name_type": {
+			Type:         schema.TypeString,
+			ForceNew:     true,
+			Optional:     true,
+			Default:      "lan-ip",
+			Description:  "Node name type of Cluster, the available values include: 'lan-ip' and 'hostname', Default is 'lan-ip'.",
+			ValidateFunc: validateAllowedStringValue(TKE_CLUSTER_NODE_NAME_TYPE),
+		},
+		"network_type": {
+			Type:         schema.TypeString,
+			ForceNew:     true,
+			Optional:     true,
+			Default:      "GR",
+			ValidateFunc: validateAllowedStringValue(TKE_CLUSTER_NETWORK_TYPE),
+			Description:  "Cluster network type, GR or VPC-CNI. Default is GR.",
+		},
+		"is_non_static_ip_mode": {
+			Type:        schema.TypeBool,
+			ForceNew:    true,
+			Optional:    true,
+			Default:     false,
+			Description: "Indicates whether static ip mode is enabled. Default is false.",
+		},
+		"deletion_protection": {
+			Type:        schema.TypeBool,
+			Optional:    true,
+			Default:     false,
+			Description: "Indicates whether cluster deletion protection is enabled. Default is false.",
+		},
+		"kube_proxy_mode": {
+			Type:         schema.TypeString,
+			Optional:     true,
+			Default:      "",
+			ValidateFunc: validateAllowedStringValue(TKE_CLUSTER_KUBE_PROXY_MODE),
+			Description: "Cluster kube-proxy mode, the available values include: 'kube-proxy-bpf'. Default is not set." +
+				"When set to kube-proxy-bpf, cluster version greater than 1.14 and with TKE-optimized kernel is required.",
+		},
 		"vpc_id": {
 			Type:         schema.TypeString,
 			ForceNew:     true,
@@ -456,10 +532,13 @@ func resourceTencentCloudTkeCluster() *schema.Resource {
 		"cluster_cidr": {
 			Type:        schema.TypeString,
 			ForceNew:    true,
-			Required:    true,
+			Optional:    true,
 			Description: "A network address block of the cluster. Different from vpc cidr and cidr of other clusters within this vpc. Must be in  10./192.168/172.[16-31] segments.",
 			ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
 				value := v.(string)
+				if value == "" {
+					return
+				}
 				_, ipnet, err := net.ParseCIDR(value)
 				if err != nil {
 					errors = append(errors, fmt.Errorf("%q must contain a valid CIDR, got error parsing: %s", k, err))
@@ -473,16 +552,16 @@ func resourceTencentCloudTkeCluster() *schema.Resource {
 					errors = append(errors, fmt.Errorf("%q must be a network segment", k))
 					return
 				}
-				if !strings.HasPrefix(value, "10.") && !strings.HasPrefix(value, "192.168.") && !strings.HasPrefix(value, "172.") {
-					errors = append(errors, fmt.Errorf("%q must in 10. | 192.168. | 172.[16-31]", k))
+				if !strings.HasPrefix(value, "9.") && !strings.HasPrefix(value, "10.") && !strings.HasPrefix(value, "192.168.") && !strings.HasPrefix(value, "172.") {
+					errors = append(errors, fmt.Errorf("%q must in 9. | 10. | 192.168. | 172.[16-31]", k))
 					return
 				}
 
 				if strings.HasPrefix(value, "172.") {
 					nextNo := strings.Split(value, ".")[1]
 					no, _ := strconv.ParseInt(nextNo, 10, 64)
 					if no < 16 || no > 31 {
-						errors = append(errors, fmt.Errorf("%q must in 10. | 192.168. | 172.[16-31]", k))
+						errors = append(errors, fmt.Errorf("%q must in 9.0 | 10. | 192.168. | 172.[16-31]", k))
 						return
 					}
 				}
@@ -530,6 +609,69 @@ func resourceTencentCloudTkeCluster() *schema.Resource {
 			},
 			Description: "The maximum number of services in the cluster. Default is 256. Must be a multiple of 16.",
 		},
+		"service_cidr": {
+			Type:        schema.TypeString,
+			ForceNew:    true,
+			Optional:    true,
+			Description: "A network address block of the service. Different from vpc cidr and cidr of other clusters within this vpc. Must be in  10./192.168/172.[16-31] segments.",
+			ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
+				value := v.(string)
+				if value == "" {
+					return
+				}
+				_, ipnet, err := net.ParseCIDR(value)
+				if err != nil {
+					errors = append(errors, fmt.Errorf("%q must contain a valid CIDR, got error parsing: %s", k, err))
+					return
+				}
+				if ipnet == nil || value != ipnet.String() {
+					errors = append(errors, fmt.Errorf("%q must contain a valid network CIDR, expected %q, got %q", k, ipnet, value))
+					return
+				}
+				if !strings.Contains(value, "/") {
+					errors = append(errors, fmt.Errorf("%q must be a network segment", k))
+					return
+				}
+				if !strings.HasPrefix(value, "9.") && !strings.HasPrefix(value, "10.") && !strings.HasPrefix(value, "192.168.") && !strings.HasPrefix(value, "172.") {
+					errors = append(errors, fmt.Errorf("%q must in 9. | 10. | 192.168. | 172.[16-31]", k))
+					return
+				}
+
+				if strings.HasPrefix(value, "172.") {
+					nextNo := strings.Split(value, ".")[1]
+					no, _ := strconv.ParseInt(nextNo, 10, 64)
+					if no < 16 || no > 31 {
+						errors = append(errors, fmt.Errorf("%q must in 9. | 10. | 192.168. | 172.[16-31]", k))
+						return
+					}
+				}
+				return
+			},
+		},
+		"eni_subnet_ids": {
+			Type:     schema.TypeList,
+			Optional: true,
+			Elem:     &schema.Schema{Type: schema.TypeString},
+			Description: "Subnet Ids for cluster with VPC-CNI network mode." +
+				" This field can only set when field `network_type` is 'VPC-CNI'." +
+				" `eni_subnet_ids` can not empty once be set.",
+		},
+		"claim_expired_seconds": {
+			Type:     schema.TypeInt,
+			Optional: true,
+			Default:  300,
+			Description: "Claim expired seconds to recycle ENI." +
+				" This field can only set when field `network_type` is 'VPC-CNI'." +
+				" `claim_expired_seconds` must greater or equal than 300 and less than 15768000.",
+			ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
+				value := v.(int)
+				if value < 300 || value > 15768000 {
+					errors = append(errors, fmt.Errorf("%q must greater or equal than 300 and less than 15768000", k))
+					return
+				}
+				return
+			},
+		},
 		"master_config": {
 			Type:     schema.TypeList,
 			ForceNew: true,
@@ -864,27 +1006,75 @@ func resourceTencentCloudTkeClusterCreate(d *schema.ResourceData, meta interface
 		basic.ClusterDescription = v.(string)
 	}
 
-	advanced.ContainerRuntime = d.Get("container_runtime").(string)
 	advanced.Ipvs = d.Get("cluster_ipvs").(bool)
-
+	advanced.AsEnabled = d.Get("cluster_as_enabled").(bool)
+	advanced.ContainerRuntime = d.Get("container_runtime").(string)
+	advanced.NodeNameType = d.Get("node_name_type").(string)
+	advanced.NetworkType = d.Get("network_type").(string)
+	advanced.IsNonStaticIpMode = d.Get("is_non_static_ip_mode").(bool)
+	advanced.DeletionProtection = d.Get("deletion_protection").(bool)
+	advanced.KubeProxyMode = d.Get("kube_proxy_mode").(string)
+
+	if extraArgs, ok := d.GetOk("cluster_extra_args"); ok {
+		extraArgList := extraArgs.([]interface{})
+		for index := range extraArgList {
+			extraArg := extraArgList[index].(map[string]interface{})
+			if apiserverArgs, exist := extraArg["kube_apiserver"]; exist {
+				args := apiserverArgs.([]interface{})
+				for index := range args {
+					advanced.ExtraArgs.KubeAPIServer = append(advanced.ExtraArgs.KubeAPIServer, args[index].(string))
+				}
+			}
+			if cmArgs, exist := extraArg["kube_controller_manager"]; exist {
+				args := cmArgs.([]interface{})
+				for index := range args {
+					advanced.ExtraArgs.KubeControllerManager = append(advanced.ExtraArgs.KubeControllerManager, args[index].(string))
+				}
+			}
+			if schedulerArgs, exist := extraArg["kube_scheduler"]; exist {
+				args := schedulerArgs.([]interface{})
+				for index := range args {
+					advanced.ExtraArgs.KubeScheduler = append(advanced.ExtraArgs.KubeScheduler, args[index].(string))
+				}
+			}
+		}
+	}
 	cidrSet.ClusterCidr = d.Get("cluster_cidr").(string)
 	cidrSet.IgnoreClusterCidrConflict = d.Get("ignore_cluster_cidr_conflict").(bool)
 	cidrSet.MaxClusterServiceNum = int64(d.Get("cluster_max_service_num").(int))
 	cidrSet.MaxNodePodNum = int64(d.Get("cluster_max_pod_num").(int))
+	cidrSet.ServiceCIDR = d.Get("service_cidr").(string)
+	cidrSet.ClaimExpiredSeconds = int64(d.Get("claim_expired_seconds").(int))
+
+	if advanced.NetworkType == TKE_CLUSTER_NETWORK_TYPE_VPC_CNI {
+		// VPC-CNI cluster need to set eni subnet and service cidr.
+		eniSubnetIdList := d.Get("eni_subnet_ids").([]interface{})
+		for index := range eniSubnetIdList {
+			subnetId := eniSubnetIdList[index].(string)
+			cidrSet.EniSubnetIds = append(cidrSet.EniSubnetIds, subnetId)
+		}
+		if cidrSet.ServiceCIDR == "" || len(cidrSet.EniSubnetIds) == 0 {
+			return fmt.Errorf("`service_cidr` must be set and `eni_subnet_ids` must be set when cluster `network_type` is VPC-CNI.")
+		}
+	} else {
+		// GR cluster
+		if cidrSet.ClusterCidr == "" {
+			return fmt.Errorf("`service_cidr` must be set when cluster `network_type` is GR")
+		}
+		items := strings.Split(cidrSet.ClusterCidr, "/")
+		if len(items) != 2 {
+			return fmt.Errorf("`cluster_cidr` must be network segment ")
+		}
 
-	items := strings.Split(cidrSet.ClusterCidr, "/")
-	if len(items) != 2 {
-		return fmt.Errorf("`cluster_cidr` must be network segment ")
-	}
-
-	bitNumber, err := strconv.ParseInt(items[1], 10, 64)
+		bitNumber, err := strconv.ParseInt(items[1], 10, 64)
 
-	if err != nil {
-		return fmt.Errorf("`cluster_cidr` must be network segment ")
-	}
+		if err != nil {
+			return fmt.Errorf("`cluster_cidr` must be network segment ")
+		}
 
-	if math.Pow(2, float64(32-bitNumber)) <= float64(cidrSet.MaxNodePodNum) {
-		return fmt.Errorf("`cluster_cidr` Network segment range is too small, can not cover cluster_max_service_num")
+		if math.Pow(2, float64(32-bitNumber)) <= float64(cidrSet.MaxNodePodNum) {
+			return fmt.Errorf("`cluster_cidr` Network segment range is too small, can not cover cluster_max_service_num")
+		}
 	}
 
 	if masters, ok := d.GetOk("master_config"); ok {