Skip to content

Commit 11bef0d

Browse files
daimaxiaxiedaimaxiaxie
committed
implement part of the LaunchTemplateProvider
1 parent 4bcdba3 commit 11bef0d

File tree

2 files changed

+208
-0
lines changed

2 files changed

+208
-0
lines changed

‎pkg/providers/launchtemplate/launchtemplate.go‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
Copyright 2024 The CloudPilot AI Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package launchtemplate
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"net"
23+
"sync"
24+
"time"
25+
26+
ecs "github.com/alibabacloud-go/ecs-20140526/v4/client"
27+
util "github.com/alibabacloud-go/tea-utils/v2/service"
28+
"github.com/alibabacloud-go/tea/tea"
29+
"github.com/patrickmn/go-cache"
30+
"github.com/samber/lo"
31+
"go.uber.org/multierr"
32+
"sigs.k8s.io/controller-runtime/pkg/log"
33+
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
34+
"sigs.k8s.io/karpenter/pkg/cloudprovider"
35+
"sigs.k8s.io/karpenter/pkg/utils/pretty"
36+
37+
"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/apis/v1alpha1"
38+
"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/operator/options"
39+
"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/providers/securitygroup"
40+
"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/providers/vswitch"
41+
"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/utils"
42+
)
43+
44+
type Provider interface {
45+
EnsureAll(context.Context, *v1alpha1.ECSNodeClass, *karpv1.NodeClaim,
46+
[]*cloudprovider.InstanceType, string, map[string]string) ([]*LaunchTemplate, error)
47+
DeleteAll(context.Context, *v1alpha1.ECSNodeClass) error
48+
InvalidateCache(context.Context, string, string)
49+
}
50+
51+
type LaunchTemplate struct {
52+
Name string
53+
InstanceTypes []*cloudprovider.InstanceType
54+
ImageID string
55+
}
56+
57+
type DefaultProvider struct {
58+
sync.Mutex
59+
region string
60+
ecsapi ecs.Client
61+
cache *cache.Cache
62+
cm *pretty.ChangeMonitor
63+
}
64+
65+
// TODO: add imagefamily args later
66+
func NewDefaultProvider(ctx context.Context, cache *cache.Cache, region string, ecsapi ecs.Client,
67+
securityGroupProvider securitygroup.Provider, subnetProvider vswitch.Provider,
68+
caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *DefaultProvider {
69+
l := &DefaultProvider{
70+
region: region,
71+
ecsapi: ecsapi,
72+
cache: cache,
73+
cm: pretty.NewChangeMonitor(),
74+
}
75+
l.cache.OnEvicted(l.cachedEvictedFunc(ctx))
76+
go func() {
77+
// only hydrate cache once elected leader
78+
select {
79+
case <-startAsync:
80+
case <-ctx.Done():
81+
return
82+
}
83+
l.hydrateCache(ctx)
84+
}()
85+
return l
86+
}
87+
88+
func (p *DefaultProvider) EnsureAll(ctx context.Context, nodeClass *v1alpha1.ECSNodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, tags map[string]string) ([]*LaunchTemplate, error) {
89+
//TODO implement me
90+
panic("implement me")
91+
}
92+
93+
func (p *DefaultProvider) DeleteAll(ctx context.Context, nodeClass *v1alpha1.ECSNodeClass) error {
94+
clusterName := options.FromContext(ctx).ClusterName
95+
tags := []*ecs.DescribeLaunchTemplatesRequestTemplateTag{{
96+
Key: tea.String(v1alpha1.TagManagedLaunchTemplate),
97+
Value: tea.String(clusterName),
98+
}, {
99+
Key: tea.String(v1alpha1.LabelNodeClass),
100+
Value: tea.String(nodeClass.Name),
101+
}}
102+
103+
var ltNames []*string
104+
if err := p.describeLaunchTemplates(&ecs.DescribeLaunchTemplatesRequest{RegionId: tea.String(p.region), TemplateTag: tags}, func(lt *ecs.DescribeLaunchTemplatesResponseBodyLaunchTemplateSetsLaunchTemplateSet) {
105+
ltNames = append(ltNames, lt.LaunchTemplateName)
106+
}); err != nil {
107+
log.FromContext(ctx).Error(err, "describe LaunchTemplates error")
108+
return fmt.Errorf("fetching launch templates, %w", err)
109+
}
110+
111+
var deleteErr error
112+
for _, name := range ltNames {
113+
_, err := p.ecsapi.DeleteLaunchTemplateWithOptions(&ecs.DeleteLaunchTemplateRequest{RegionId: tea.String(p.region), LaunchTemplateName: name}, &util.RuntimeOptions{})
114+
deleteErr = multierr.Append(deleteErr, err)
115+
}
116+
if len(ltNames) > 0 {
117+
log.FromContext(ctx).WithValues("launchTemplates", utils.PrettySlice(lo.FromSlicePtr(ltNames), 5)).V(1).Info("deleted launch templates")
118+
}
119+
if deleteErr != nil {
120+
return fmt.Errorf("deleting launch templates, %w", deleteErr)
121+
}
122+
return nil
123+
}
124+
125+
// InvalidateCache deletes a launch template from cache if it exists
126+
func (p *DefaultProvider) InvalidateCache(ctx context.Context, ltName string, ltID string) {
127+
ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("launch-template-name", ltName, "launch-template-id", ltID))
128+
p.Lock()
129+
defer p.Unlock()
130+
defer p.cache.OnEvicted(p.cachedEvictedFunc(ctx))
131+
p.cache.OnEvicted(nil)
132+
log.FromContext(ctx).V(1).Info("invalidating launch template in the cache because it no longer exists")
133+
p.cache.Delete(ltName)
134+
}
135+
136+
// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache.
137+
// Any error during hydration will result in a panic
138+
func (p *DefaultProvider) hydrateCache(ctx context.Context) {
139+
clusterName := options.FromContext(ctx).ClusterName
140+
ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("tag-key", v1alpha1.TagManagedLaunchTemplate, "tag-value", clusterName))
141+
tags := []*ecs.DescribeLaunchTemplatesRequestTemplateTag{{
142+
Key: tea.String(v1alpha1.TagManagedLaunchTemplate),
143+
Value: tea.String(clusterName),
144+
}}
145+
if err := p.describeLaunchTemplates(&ecs.DescribeLaunchTemplatesRequest{RegionId: tea.String(p.region), TemplateTag: tags}, func(lt *ecs.DescribeLaunchTemplatesResponseBodyLaunchTemplateSetsLaunchTemplateSet) {
146+
p.cache.SetDefault(*lt.LaunchTemplateName, lt)
147+
}); err != nil {
148+
log.FromContext(ctx).Error(err, "unable to hydrate the AWS launch template cache")
149+
}
150+
log.FromContext(ctx).WithValues("count", p.cache.ItemCount()).V(1).Info("hydrated launch template cache")
151+
}
152+
153+
func (p *DefaultProvider) cachedEvictedFunc(ctx context.Context) func(string, interface{}) {
154+
return func(key string, lt interface{}) {
155+
p.Lock()
156+
defer p.Unlock()
157+
if _, expiration, _ := p.cache.GetWithExpiration(key); expiration.After(time.Now()) {
158+
return
159+
}
160+
launchTemplate := lt.(*ecs.DescribeLaunchTemplatesResponseBodyLaunchTemplateSetsLaunchTemplateSet)
161+
// no guarantee of deletion
162+
if _, err := p.ecsapi.DeleteLaunchTemplateWithOptions(&ecs.DeleteLaunchTemplateRequest{RegionId: tea.String(p.region), LaunchTemplateId: launchTemplate.LaunchTemplateId, LaunchTemplateName: launchTemplate.LaunchTemplateName}, &util.RuntimeOptions{}); err != nil {
163+
log.FromContext(ctx).WithValues("launch-template", launchTemplate).Error(err, "failed to delete launch template") // If the LaunchTemplate does not exist, no error is returned.
164+
return
165+
}
166+
log.FromContext(ctx).WithValues("id", *launchTemplate.LaunchTemplateId, "name", *launchTemplate.LaunchTemplateName).V(1).Info("deleted launch template")
167+
}
168+
}
169+
170+
func (p *DefaultProvider) describeLaunchTemplates(request *ecs.DescribeLaunchTemplatesRequest, process func(*ecs.DescribeLaunchTemplatesResponseBodyLaunchTemplateSetsLaunchTemplateSet)) error {
171+
runtime := &util.RuntimeOptions{}
172+
request.PageSize = tea.Int32(50)
173+
for pageNumber := int32(1); pageNumber < 500; pageNumber++ {
174+
request.PageNumber = tea.Int32(pageNumber)
175+
output, err := p.ecsapi.DescribeLaunchTemplatesWithOptions(request, runtime)
176+
if err != nil {
177+
return err
178+
} else if output == nil || output.Body == nil || output.Body.TotalCount == nil || output.Body.LaunchTemplateSets == nil {
179+
return fmt.Errorf("unexpected null value was returned")
180+
}
181+
182+
for i := range output.Body.LaunchTemplateSets.LaunchTemplateSet {
183+
process(output.Body.LaunchTemplateSets.LaunchTemplateSet[i])
184+
}
185+
186+
if *output.Body.TotalCount < pageNumber*50 || len(output.Body.LaunchTemplateSets.LaunchTemplateSet) < 50 {
187+
break
188+
}
189+
}
190+
return nil
191+
}

‎pkg/utils/utils.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package utils
1919
import (
2020
"fmt"
2121
"regexp"
22+
"strings"
2223
"time"
2324

2425
"sigs.k8s.io/karpenter/pkg/cloudprovider"
@@ -63,3 +64,19 @@ func GetAllSingleValuedRequirementLabels(instanceType *cloudprovider.InstanceTyp
6364
}
6465
return labels
6566
}
67+
68+
// PrettySlice truncates a slice after a certain number of max items to ensure
69+
// that the Slice isn't too long
70+
func PrettySlice[T any](s []T, maxItems int) string {
71+
var sb strings.Builder
72+
for i, elem := range s {
73+
if i > maxItems-1 {
74+
fmt.Fprintf(&sb, " and %d other(s)", len(s)-i)
75+
break
76+
} else if i > 0 {
77+
fmt.Fprint(&sb, ", ")
78+
}
79+
fmt.Fprint(&sb, elem)
80+
}
81+
return sb.String()
82+
}

0 commit comments

Comments
 (0)