Merge pull request #44 from daimaxiaxie/impl-launch-template-provider-ensure-all

implement LaunchTemplateProvider.EnsureAll

Author: jwcesign

config/components/crds/karpenter.k8s.alicloud_ecsnodeclasses.yaml

@@ -278,6 +278,73 @@ spec:
                 - message: '''name'' is mutually exclusive, cannot be set with a combination
                     of other fields in securityGroupSelectorTerms'
                   rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))'
+              systemDisk:
+                description: SystemDisk to be applied to provisioned nodes.
+                properties:
+                  autoSnapshotPolicyId:
+                    description: The ID of the automatic snapshot policy to apply
+                      to the system disk.
+                    type: string
+                  burstingEnabled:
+                    description: Specifies whether to enable the performance burst
+                      feature for the system disk
+                    type: boolean
+                  category:
+                    description: |-
+                      The category of the system disk (for example, cloud or cloud_ssd).
+                      Only one of the following: "cloud", "cloud_efficiency", "cloud_ssd", "cloud_essd", "cloud_auto", and "cloud_essd_entry"
+                    enum:
+                    - cloud
+                    - cloud_efficiency
+                    - cloud_ssd
+                    - cloud_essd
+                    - cloud_auto
+                    - cloud_essd_entry
+                    type: string
+                  diskName:
+                    description: |-
+                      The name of the system disk.
+                      The name must be 2 to 128 characters in length.
+                      The name must start with a letter and cannot start with http:// or https://.
+                      The name can contain letters, digits, colons (:), underscores (_), and hyphens (-).
+                    pattern: ^[A-Za-z][A-Za-z0-9:_-]*$
+                    type: string
+                    x-kubernetes-validations:
+                    - message: format invalid
+                      rule: '!self.startsWith(''http'') && self.size() >= 2 && self.size()
+                        <= 128'
+                  performanceLevel:
+                    description: |-
+                      The performance level of the ESSD to use as the system disk. Default value: PL0.
+                      Valid values:
+                        * PL0: A single ESSD can deliver up to 10,000 random read/write IOPS.
+                        * PL1: A single ESSD can deliver up to 50,000 random read/write IOPS.
+                        * PL2: A single ESSD can deliver up to 100,000 random read/write IOPS.
+                        * PL3: A single ESSD can deliver up to 1,000,000 random read/write IOPS.
+                    enum:
+                    - PL0
+                    - PL1
+                    - PL2
+                    - PL3
+                    type: string
+                  size:
+                    description: |-
+                      The size of the system disk. Unit: GiB.
+                      Valid values:
+                        * If you set Category to cloud: 20 to 500.
+                        * If you set Category to other disk categories: 20 to 2048.
+                    format: int32
+                    type: integer
+                    x-kubernetes-validations:
+                    - message: size invalid
+                      rule: self >= 20
+                type: object
+              userData:
+                description: |-
+                  UserData to be applied to the provisioned nodes.
+                  It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
+                  this UserData to ensure nodes are being provisioned with the correct configuration.
+                type: string
               vSwitchSelectorTerms:
                 description: VSwitchSelectorTerms is a list of or vSwitch selector
                   terms. The terms are ORed.
@@ -376,6 +443,57 @@ spec:
                   - type
                   type: object
                 type: array
+              images:
+                description: |-
+                  Image contains the current image that are available to the
+                  cluster under the Image selectors.
+                items:
+                  description: Image contains resolved image selector values utilized
+                    for node launch
+                  properties:
+                    id:
+                      description: ID of the Image
+                      type: string
+                    name:
+                      description: Name of the Image
+                      type: string
+                    requirements:
+                      description: Requirements of the Image to be utilized on an
+                        instance type
+                      items:
+                        description: |-
+                          A node selector requirement is a selector that contains values, a key, and an operator
+                          that relates the key and values.
+                        properties:
+                          key:
+                            description: The label key that the selector applies to.
+                            type: string
+                          operator:
+                            description: |-
+                              Represents a key's relationship to a set of values.
+                              Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+                            type: string
+                          values:
+                            description: |-
+                              An array of string values. If the operator is In or NotIn,
+                              the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                              the values array must be empty. If the operator is Gt or Lt, the values
+                              array must have a single element, which will be interpreted as an integer.
+                              This array is replaced during a strategic merge patch.
+                            items:
+                              type: string
+                            type: array
+                            x-kubernetes-list-type: atomic
+                        required:
+                        - key
+                        - operator
+                        type: object
+                      type: array
+                  required:
+                  - id
+                  - requirements
+                  type: object
+                type: array
               securityGroups:
                 description: |-
                   SecurityGroups contains the current Security Groups values that are available to the

go.mod

@@ -4,7 +4,7 @@ go 1.23
 
 require (
 	github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.10
-	github.com/alibabacloud-go/ecs-20140526/v4 v4.26.1
+	github.com/alibabacloud-go/ecs-20140526/v4 v4.26.4
 	github.com/alibabacloud-go/tea v1.2.2
 	github.com/alibabacloud-go/tea-utils/v2 v2.0.6
 	github.com/alibabacloud-go/vpc-20160428/v6 v6.10.4
@@ -28,13 +28,7 @@ require (
 require (
 	contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect
 	contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect
-	github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.6 // indirect
 	github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5 // indirect
-	github.com/alibabacloud-go/darabonba-array v0.1.0 // indirect
-	github.com/alibabacloud-go/darabonba-encode-util v0.0.2 // indirect
-	github.com/alibabacloud-go/darabonba-map v0.0.2 // indirect
-	github.com/alibabacloud-go/darabonba-signature-util v0.0.7 // indirect
-	github.com/alibabacloud-go/darabonba-string v1.0.2 // indirect
 	github.com/alibabacloud-go/debug v1.0.1 // indirect
 	github.com/alibabacloud-go/endpoint-util v1.1.0 // indirect
 	github.com/alibabacloud-go/openapi-util v0.1.0 // indirect

go.sum

@@ -65,8 +65,8 @@ github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6p
 github.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc=
 github.com/alibabacloud-go/debug v1.0.1 h1:MsW9SmUtbb1Fnt3ieC6NNZi6aEwrXfDksD4QA6GSbPg=
 github.com/alibabacloud-go/debug v1.0.1/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc=
-github.com/alibabacloud-go/ecs-20140526/v4 v4.26.1 h1:H7iwq9BkcEBSc5cILRI0mz/G5liUCYzQdAtnpqYhaZ0=
-github.com/alibabacloud-go/ecs-20140526/v4 v4.26.1/go.mod h1:+OL8LuKrMnvZHxGETlNFY7NyYC4iba6zeVJo2OzX+zI=
+github.com/alibabacloud-go/ecs-20140526/v4 v4.26.4 h1:6SGPir8HT9YolNjTiDc1RBsU2BhQ3sBLy+kecxqDV7M=
+github.com/alibabacloud-go/ecs-20140526/v4 v4.26.4/go.mod h1:RAjofcYEeLI1zDF/bhyyHI/fRCMnPh7ZI/Swy2mEiXA=
 github.com/alibabacloud-go/endpoint-util v1.1.0 h1:r/4D3VSw888XGaeNpP994zDUaxdgTSHBbVfZlzf6b5Q=
 github.com/alibabacloud-go/endpoint-util v1.1.0/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE=
 github.com/alibabacloud-go/openapi-util v0.1.0 h1:0z75cIULkDrdEhkLWgi9tnLe+KhAFE/r5Pb3312/eAY=

pkg/apis/v1alpha1/ecsnodeclass.go

@@ -53,6 +53,11 @@ type ECSNodeClassSpec struct {
 	// +kubebuilder:validation:MaxItems:=30
 	// +required
 	ImageSelectorTerms []ImageSelectorTerm `json:"imageSelectorTerms" hash:"ignore"`
+	// UserData to be applied to the provisioned nodes.
+	// It must be in the appropriate format based on the ImageFamily in use. Karpenter will merge certain fields into
+	// this UserData to ensure nodes are being provisioned with the correct configuration.
+	// +optional
+	UserData *string `json:"userData,omitempty"`
 	// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes.
 	// They are a subset of the upstream types, recognizing not all options may be supported.
 	// Wherever possible, the types and names should reflect the upstream kubelet types.
@@ -61,6 +66,9 @@ type ECSNodeClassSpec struct {
 	// +kubebuilder:validation:XValidation:message="evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft",rule="has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true"
 	// +optional
 	KubeletConfiguration *KubeletConfiguration `json:"kubeletConfiguration,omitempty"`
+	// SystemDisk to be applied to provisioned nodes.
+	// +optional
+	SystemDisk *SystemDisk `json:"systemDisk,omitempty"`
 }
 
 // VSwitchSelectorTerm defines selection logic for a vSwitch used by Karpenter to launch nodes.
@@ -196,6 +204,43 @@ type KubeletConfiguration struct {
 	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"`
 }
 
+type SystemDisk struct {
+	// The category of the system disk (for example, cloud or cloud_ssd).
+	// Only one of the following: "cloud", "cloud_efficiency", "cloud_ssd", "cloud_essd", "cloud_auto", and "cloud_essd_entry"
+	// +kubebuilder:validation:Enum:={cloud,cloud_efficiency,cloud_ssd,cloud_essd,cloud_auto,cloud_essd_entry}
+	// +optional
+	Category *string `json:"category,omitempty"`
+	// The size of the system disk. Unit: GiB.
+	// Valid values:
+	//   * If you set Category to cloud: 20 to 500.
+	//   * If you set Category to other disk categories: 20 to 2048.
+	//
+	// +kubebuilder:validation:XValidation:message="size invalid",rule="self >= 20"
+	// +optional
+	Size *int32 `json:"size,omitempty"`
+	// The name of the system disk.
+	// The name must be 2 to 128 characters in length.
+	// The name must start with a letter and cannot start with http:// or https://.
+	// The name can contain letters, digits, colons (:), underscores (_), and hyphens (-).
+	// +kubebuilder:validation:XValidation:message="format invalid",rule="!self.startsWith('http') && self.size() >= 2 && self.size() <= 128"
+	// +kubebuilder:validation:Pattern="^[A-Za-z][A-Za-z0-9:_-]*$"
+	// +optional
+	DiskName *string `json:"diskName,omitempty"`
+	// The performance level of the ESSD to use as the system disk. Default value: PL0.
+	// Valid values:
+	//   * PL0: A single ESSD can deliver up to 10,000 random read/write IOPS.
+	//   * PL1: A single ESSD can deliver up to 50,000 random read/write IOPS.
+	//   * PL2: A single ESSD can deliver up to 100,000 random read/write IOPS.
+	//   * PL3: A single ESSD can deliver up to 1,000,000 random read/write IOPS.
+	// +kubebuilder:validation:Enum:={PL0,PL1,PL2,PL3}
+	PerformanceLevel *string `json:"performanceLevel,omitempty"`
+	// The ID of the automatic snapshot policy to apply to the system disk.
+	AutoSnapshotPolicyID *string `json:"autoSnapshotPolicyId,omitempty"`
+	// Specifies whether to enable the performance burst feature for the system disk
+	BurstingEnabled *bool `json:"burstingEnabled,omitempty"`
+	// TODO: add ProvisionedIops or Iops
+}
+
 // ECSNodeClass is the Schema for the ECSNodeClass API
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:resource:path=ecsnodeclasses,scope=Cluster,categories=karpenter,shortName={ecsnc,ecsncs}

pkg/apis/v1alpha1/ecsnodeclass_status.go

@@ -18,6 +18,7 @@ package v1alpha1
 
 import (
 	"github.com/awslabs/operatorpkg/status"
+	corev1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -46,6 +47,19 @@ type SecurityGroup struct {
 	Name string `json:"name,omitempty"`
 }
 
+// Image contains resolved image selector values utilized for node launch
+type Image struct {
+	// ID of the Image
+	// +required
+	ID string `json:"id"`
+	// Name of the Image
+	// +optional
+	Name string `json:"name,omitempty"`
+	// Requirements of the Image to be utilized on an instance type
+	// +required
+	Requirements []corev1.NodeSelectorRequirement `json:"requirements"`
+}
+
 // ECSNodeClassStatus contains the resolved state of the ECSNodeClass
 type ECSNodeClassStatus struct {
 	// VSwitches contains the current VSwitch values that are available to the
@@ -56,6 +70,10 @@ type ECSNodeClassStatus struct {
 	// cluster under the SecurityGroups selectors.
 	// +optional
 	SecurityGroups []SecurityGroup `json:"securityGroups,omitempty"`
+	// Image contains the current image that are available to the
+	// cluster under the Image selectors.
+	// +optional
+	Images []Image `json:"images,omitempty"`
 	// Conditions contains signals for health and readiness
 	// +optional
 	Conditions []status.Condition `json:"conditions,omitempty"`

pkg/providers/imagefamily/aliyun3.go

@@ -16,12 +16,35 @@ limitations under the License.
 
 package imagefamily
 
-import "context"
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/karpenter/pkg/cloudprovider"
+
+	"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/apis/v1alpha1"
+)
 
 type Aliyun3 struct {
+	*Options
+}
+
+func (a Aliyun3) UserData(kubeletConfig *v1alpha1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, instanceTypes []*cloudprovider.InstanceType, customUserData *string) string {
+	// in-cluster user data
+	// curl http://aliacs-k8s-{region}.oss-{region}-internal.aliyuncs.com/public/pkg/run/attach/{clusterversion}/attach_node.sh | bash -s -- --token **** --endpoint 1.x.x.x --cluster-dns 1.x.x.x --cluster-id ...
+	//
+	// token: secret/bootstrap-token
+
+	//TODO implement me
+	panic("implement me")
 }
 
 func (a Aliyun3) DescribeImageQuery(ctx context.Context, k8sVersion string, imageVersion string) ([]DescribeImageQuery, error) {
 	//TODO implement me
 	panic("implement me")
 }
+
+func (a Aliyun3) DefaultSystemDisk() *v1alpha1.SystemDisk {
+	//TODO implement me
+	panic("implement me")
+}

pkg/providers/imagefamily/custom.go

@@ -18,22 +18,28 @@ package imagefamily
 
 import (
 	"context"
+	"encoding/base64"
 
+	"github.com/alibabacloud-go/tea/tea"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/karpenter/pkg/cloudprovider"
 
 	"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/apis/v1alpha1"
 )
 
 type Custom struct {
+	*Options
 }
 
 // UserData returns the default userdata script for the Image Family
-func (c Custom) UserData(_ *v1alpha1.KubeletConfiguration, _ []corev1.Taint, _ map[string]string, _ *string, _ []*cloudprovider.InstanceType, customUserData *string) {
-	//TODO implement me
-	panic("implement me")
+func (c Custom) UserData(kubeletConfig *v1alpha1.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, instanceTypes []*cloudprovider.InstanceType, customUserData *string) string {
+	return base64.StdEncoding.EncodeToString([]byte(tea.StringValue(customUserData)))
 }
 
 func (c Custom) DescribeImageQuery(_ context.Context, _ string, _ string) ([]DescribeImageQuery, error) {
 	return []DescribeImageQuery{}, nil
 }
+
+func (c Custom) DefaultSystemDisk() *v1alpha1.SystemDisk {
+	return nil
+}

pkg/providers/imagefamily/image.go

@@ -90,7 +90,7 @@ func (p *DefaultProvider) DescribeImageQueries(ctx context.Context, nodeClass *v
 		if err != nil {
 			return nil, fmt.Errorf("getting kubernetes version, %w", err)
 		}
-		imageFamily := GetImageFamily(v1alpha1.ImageFamilyFromAlias(term.Alias))
+		imageFamily := GetImageFamily(v1alpha1.ImageFamilyFromAlias(term.Alias), nil)
 		query, err := imageFamily.DescribeImageQuery(ctx, kubernetesVersion, v1alpha1.ImageVersionFromAlias(term.Alias))
 		if err != nil {
 			return nil, err
@@ -204,15 +204,3 @@ func (p *DefaultProvider) describeImages(request *ecs.DescribeImagesRequest, pro
 	}
 	return nil
 }
-
-// TODO: move it to resolver.go
-func GetImageFamily(imageFamily string) ImageFamily {
-	switch imageFamily {
-	case v1alpha1.ImageFamilyCustom:
-		return &Custom{}
-	case v1alpha1.ImageFamilyAliyun3:
-		return &Aliyun3{}
-	default:
-		return &Aliyun3{}
-	}
-}