Skip to content

Commit e942a0f

Browse files
shraddhabangshraddhabang
authored
Update rule management to avoid sporadic 503 errors (#4039)
* Update rule management to avoid sporadic 503 errors * Update rule management to avoid sporadic 503 errors * Update rule management to avoid sporadic 503 errors * Update rule management to avoid sporadic 503 errors * Update rule management to avoid sporadic 503 errors * Update rule management to avoid sporadic 503 errors * Update rule management to avoid sporadic 503 errors
1 parent f9699da commit e942a0f

15 files changed

+1929
-68
lines changed

‎docs/install/iam_policy.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@
239239
"elasticloadbalancing:ModifyListener",
240240
"elasticloadbalancing:AddListenerCertificates",
241241
"elasticloadbalancing:RemoveListenerCertificates",
242-
"elasticloadbalancing:ModifyRule"
242+
"elasticloadbalancing:ModifyRule",
243+
"elasticloadbalancing:SetRulePriorities"
243244
],
244245
"Resource": "*"
245246
}

‎docs/install/iam_policy_cn.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@
239239
"elasticloadbalancing:ModifyListener",
240240
"elasticloadbalancing:AddListenerCertificates",
241241
"elasticloadbalancing:RemoveListenerCertificates",
242-
"elasticloadbalancing:ModifyRule"
242+
"elasticloadbalancing:ModifyRule",
243+
"elasticloadbalancing:SetRulePriorities"
243244
],
244245
"Resource": "*"
245246
}

‎docs/install/iam_policy_iso.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,8 @@
234234
"elasticloadbalancing:ModifyListener",
235235
"elasticloadbalancing:AddListenerCertificates",
236236
"elasticloadbalancing:RemoveListenerCertificates",
237-
"elasticloadbalancing:ModifyRule"
237+
"elasticloadbalancing:ModifyRule",
238+
"elasticloadbalancing:SetRulePriorities"
238239
],
239240
"Resource": "*"
240241
}

‎docs/install/iam_policy_isob.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,8 @@
234234
"elasticloadbalancing:ModifyListener",
235235
"elasticloadbalancing:AddListenerCertificates",
236236
"elasticloadbalancing:RemoveListenerCertificates",
237-
"elasticloadbalancing:ModifyRule"
237+
"elasticloadbalancing:ModifyRule",
238+
"elasticloadbalancing:SetRulePriorities"
238239
],
239240
"Resource": "*"
240241
}

‎docs/install/iam_policy_isoe.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,8 @@
234234
"elasticloadbalancing:ModifyListener",
235235
"elasticloadbalancing:AddListenerCertificates",
236236
"elasticloadbalancing:RemoveListenerCertificates",
237-
"elasticloadbalancing:ModifyRule"
237+
"elasticloadbalancing:ModifyRule",
238+
"elasticloadbalancing:SetRulePriorities"
238239
],
239240
"Resource": "*"
240241
}

‎docs/install/iam_policy_isof.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,8 @@
234234
"elasticloadbalancing:ModifyListener",
235235
"elasticloadbalancing:AddListenerCertificates",
236236
"elasticloadbalancing:RemoveListenerCertificates",
237-
"elasticloadbalancing:ModifyRule"
237+
"elasticloadbalancing:ModifyRule",
238+
"elasticloadbalancing:SetRulePriorities"
238239
],
239240
"Resource": "*"
240241
}

‎docs/install/iam_policy_us-gov.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@
239239
"elasticloadbalancing:ModifyListener",
240240
"elasticloadbalancing:AddListenerCertificates",
241241
"elasticloadbalancing:RemoveListenerCertificates",
242-
"elasticloadbalancing:ModifyRule"
242+
"elasticloadbalancing:ModifyRule",
243+
"elasticloadbalancing:SetRulePriorities"
243244
],
244245
"Resource": "*"
245246
}

‎pkg/aws/services/elbv2.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ type ELBV2 interface {
5050
DescribeRulesWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeRulesInput) (*elasticloadbalancingv2.DescribeRulesOutput, error)
5151
CreateRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.CreateRuleInput) (*elasticloadbalancingv2.CreateRuleOutput, error)
5252
DeleteRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.DeleteRuleInput) (*elasticloadbalancingv2.DeleteRuleOutput, error)
53-
ModifyRuleWithContext(ctx context.Context, inout *elasticloadbalancingv2.ModifyRuleInput) (*elasticloadbalancingv2.ModifyRuleOutput, error)
53+
ModifyRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyRuleInput) (*elasticloadbalancingv2.ModifyRuleOutput, error)
54+
SetRulePrioritiesWithContext(ctx context.Context, input *elasticloadbalancingv2.SetRulePrioritiesInput) (*elasticloadbalancingv2.SetRulePrioritiesOutput, error)
5455
RegisterTargetsWithContext(ctx context.Context, input *elasticloadbalancingv2.RegisterTargetsInput) (*elasticloadbalancingv2.RegisterTargetsOutput, error)
5556
DeregisterTargetsWithContext(ctx context.Context, input *elasticloadbalancingv2.DeregisterTargetsInput) (*elasticloadbalancingv2.DeregisterTargetsOutput, error)
5657
DescribeTrustStoresWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeTrustStoresInput) (*elasticloadbalancingv2.DescribeTrustStoresOutput, error)
@@ -163,6 +164,14 @@ func (c *elbv2Client) DeleteRuleWithContext(ctx context.Context, input *elasticl
163164
return client.DeleteRule(ctx, input)
164165
}
165166

167+
func (c *elbv2Client) SetRulePrioritiesWithContext(ctx context.Context, input *elasticloadbalancingv2.SetRulePrioritiesInput) (*elasticloadbalancingv2.SetRulePrioritiesOutput, error) {
168+
client, err := c.awsClientsProvider.GetELBv2Client(ctx, "SetRulePriorities")
169+
if err != nil {
170+
return nil, err
171+
}
172+
return client.SetRulePriorities(ctx, input)
173+
}
174+
166175
func (c *elbv2Client) CreateRuleWithContext(ctx context.Context, input *elasticloadbalancingv2.CreateRuleInput) (*elasticloadbalancingv2.CreateRuleOutput, error) {
167176
client, err := c.getClient(ctx, "CreateRule")
168177
if err != nil {

‎pkg/aws/services/elbv2_mocks.go‎

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/deploy/elbv2/listener_rule_manager.go‎

Lines changed: 86 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,29 @@ import (
66
elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
77
elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
88
"github.com/go-logr/logr"
9-
"github.com/google/go-cmp/cmp"
109
"github.com/pkg/errors"
1110
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
1211
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
1312
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
14-
elbv2equality "sigs.k8s.io/aws-load-balancer-controller/pkg/equality/elbv2"
1513
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
1614
"sigs.k8s.io/aws-load-balancer-controller/pkg/runtime"
15+
"slices"
16+
"sort"
17+
"strconv"
1718
"time"
1819
)
1920

2021
// ListenerRuleManager is responsible for create/update/delete ListenerRule resources.
2122
type ListenerRuleManager interface {
22-
Create(ctx context.Context, resLR *elbv2model.ListenerRule) (elbv2model.ListenerRuleStatus, error)
23+
Create(ctx context.Context, resLR *elbv2model.ListenerRule, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair) (elbv2model.ListenerRuleStatus, error)
2324

24-
Update(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) (elbv2model.ListenerRuleStatus, error)
25+
UpdateRules(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair) (elbv2model.ListenerRuleStatus, error)
26+
27+
UpdateRulesTags(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) (elbv2model.ListenerRuleStatus, error)
2528

2629
Delete(ctx context.Context, sdkLR ListenerRuleWithTags) error
30+
31+
SetRulePriorities(ctx context.Context, matchedResAndSDKLRsBySettings []resAndSDKListenerRulePair, unmatchedSDKLRs []ListenerRuleWithTags) error
2732
}
2833

2934
// NewDefaultListenerRuleManager constructs new defaultListenerRuleManager.
@@ -54,8 +59,8 @@ type defaultListenerRuleManager struct {
5459
waitLSExistenceTimeout time.Duration
5560
}
5661

57-
func (m *defaultListenerRuleManager) Create(ctx context.Context, resLR *elbv2model.ListenerRule) (elbv2model.ListenerRuleStatus, error) {
58-
req, err := buildSDKCreateListenerRuleInput(resLR.Spec, m.featureGates)
62+
func (m *defaultListenerRuleManager) Create(ctx context.Context, resLR *elbv2model.ListenerRule, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair) (elbv2model.ListenerRuleStatus, error) {
63+
req, err := buildSDKCreateListenerRuleInput(resLR.Spec, desiredActionsAndConditions, m.featureGates)
5964
if err != nil {
6065
return elbv2model.ListenerRuleStatus{}, err
6166
}
@@ -90,13 +95,17 @@ func (m *defaultListenerRuleManager) Create(ctx context.Context, resLR *elbv2mod
9095
return buildResListenerRuleStatus(sdkLR), nil
9196
}
9297

93-
func (m *defaultListenerRuleManager) Update(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) (elbv2model.ListenerRuleStatus, error) {
98+
func (m *defaultListenerRuleManager) UpdateRulesTags(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) (elbv2model.ListenerRuleStatus, error) {
9499
if m.featureGates.Enabled(config.ListenerRulesTagging) {
95100
if err := m.updateSDKListenerRuleWithTags(ctx, resLR, sdkLR); err != nil {
96101
return elbv2model.ListenerRuleStatus{}, err
97102
}
98103
}
99-
if err := m.updateSDKListenerRuleWithSettings(ctx, resLR, sdkLR); err != nil {
104+
return buildResListenerRuleStatus(sdkLR), nil
105+
}
106+
107+
func (m *defaultListenerRuleManager) UpdateRules(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair) (elbv2model.ListenerRuleStatus, error) {
108+
if err := m.updateSDKListenerRuleWithSettings(ctx, resLR, sdkLR, desiredActionsAndConditions); err != nil {
100109
return elbv2model.ListenerRuleStatus{}, err
101110
}
102111
return buildResListenerRuleStatus(sdkLR), nil
@@ -116,15 +125,28 @@ func (m *defaultListenerRuleManager) Delete(ctx context.Context, sdkLR ListenerR
116125
return nil
117126
}
118127

119-
func (m *defaultListenerRuleManager) updateSDKListenerRuleWithSettings(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) error {
120-
desiredActions, err := buildSDKActions(resLR.Spec.Actions, m.featureGates)
121-
if err != nil {
128+
func (m *defaultListenerRuleManager) SetRulePriorities(ctx context.Context, matchedResAndSDKLRsBySettings []resAndSDKListenerRulePair, unmatchedSDKLRs []ListenerRuleWithTags) error {
129+
req := buildSDKSetRulePrioritiesInput(matchedResAndSDKLRsBySettings, unmatchedSDKLRs)
130+
m.logger.Info("setting listener rule priorities",
131+
"rule priority pairs", req.RulePriorities)
132+
if _, err := m.elbv2Client.SetRulePrioritiesWithContext(ctx, req); err != nil {
122133
return err
123134
}
124-
desiredConditions := buildSDKRuleConditions(resLR.Spec.Conditions)
125-
if !isSDKListenerRuleSettingsDrifted(resLR.Spec, sdkLR, desiredActions, desiredConditions) {
126-
return nil
127-
}
135+
m.logger.Info("setting listener rule priorities complete",
136+
"rule priority pairs", req.RulePriorities)
137+
return nil
138+
}
139+
140+
func (m *defaultListenerRuleManager) updateSDKListenerRuleWithTags(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) error {
141+
desiredTags := m.trackingProvider.ResourceTags(resLR.Stack(), resLR, resLR.Spec.Tags)
142+
return m.taggingManager.ReconcileTags(ctx, awssdk.ToString(sdkLR.ListenerRule.RuleArn), desiredTags,
143+
WithCurrentTags(sdkLR.Tags),
144+
WithIgnoredTagKeys(m.externalManagedTags))
145+
}
146+
147+
func (m *defaultListenerRuleManager) updateSDKListenerRuleWithSettings(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair) error {
148+
desiredActions := desiredActionsAndConditions.desiredActions
149+
desiredConditions := desiredActionsAndConditions.desiredConditions
128150

129151
req := buildSDKModifyListenerRuleInput(resLR.Spec, desiredActions, desiredConditions)
130152
req.RuleArn = sdkLR.ListenerRule.RuleArn
@@ -142,27 +164,7 @@ func (m *defaultListenerRuleManager) updateSDKListenerRuleWithSettings(ctx conte
142164
return nil
143165
}
144166

145-
func (m *defaultListenerRuleManager) updateSDKListenerRuleWithTags(ctx context.Context, resLR *elbv2model.ListenerRule, sdkLR ListenerRuleWithTags) error {
146-
desiredTags := m.trackingProvider.ResourceTags(resLR.Stack(), resLR, resLR.Spec.Tags)
147-
return m.taggingManager.ReconcileTags(ctx, awssdk.ToString(sdkLR.ListenerRule.RuleArn), desiredTags,
148-
WithCurrentTags(sdkLR.Tags),
149-
WithIgnoredTagKeys(m.externalManagedTags))
150-
}
151-
152-
func isSDKListenerRuleSettingsDrifted(lrSpec elbv2model.ListenerRuleSpec, sdkLR ListenerRuleWithTags,
153-
desiredActions []elbv2types.Action, desiredConditions []elbv2types.RuleCondition) bool {
154-
155-
if !cmp.Equal(desiredActions, sdkLR.ListenerRule.Actions, elbv2equality.CompareOptionForActions()) {
156-
return true
157-
}
158-
if !cmp.Equal(desiredConditions, sdkLR.ListenerRule.Conditions, elbv2equality.CompareOptionForRuleConditions()) {
159-
return true
160-
}
161-
162-
return false
163-
}
164-
165-
func buildSDKCreateListenerRuleInput(lrSpec elbv2model.ListenerRuleSpec, featureGates config.FeatureGates) (*elbv2sdk.CreateRuleInput, error) {
167+
func buildSDKCreateListenerRuleInput(lrSpec elbv2model.ListenerRuleSpec, desiredActionsAndConditions *resLRDesiredActionsAndConditionsPair, featureGates config.FeatureGates) (*elbv2sdk.CreateRuleInput, error) {
166168
ctx := context.Background()
167169
lsARN, err := lrSpec.ListenerARN.Resolve(ctx)
168170
if err != nil {
@@ -171,12 +173,20 @@ func buildSDKCreateListenerRuleInput(lrSpec elbv2model.ListenerRuleSpec, feature
171173
sdkObj := &elbv2sdk.CreateRuleInput{}
172174
sdkObj.ListenerArn = awssdk.String(lsARN)
173175
sdkObj.Priority = awssdk.Int32(lrSpec.Priority)
174-
actions, err := buildSDKActions(lrSpec.Actions, featureGates)
175-
if err != nil {
176-
return nil, err
176+
if desiredActionsAndConditions != nil && desiredActionsAndConditions.desiredActions != nil {
177+
sdkObj.Actions = desiredActionsAndConditions.desiredActions
178+
} else {
179+
actions, err := buildSDKActions(lrSpec.Actions, featureGates)
180+
if err != nil {
181+
return nil, err
182+
}
183+
sdkObj.Actions = actions
184+
}
185+
if desiredActionsAndConditions != nil && desiredActionsAndConditions.desiredConditions != nil {
186+
sdkObj.Conditions = desiredActionsAndConditions.desiredConditions
187+
} else {
188+
sdkObj.Conditions = buildSDKRuleConditions(lrSpec.Conditions)
177189
}
178-
sdkObj.Actions = actions
179-
sdkObj.Conditions = buildSDKRuleConditions(lrSpec.Conditions)
180190
return sdkObj, nil
181191
}
182192

@@ -187,6 +197,41 @@ func buildSDKModifyListenerRuleInput(_ elbv2model.ListenerRuleSpec, desiredActio
187197
return sdkObj
188198
}
189199

200+
func buildSDKSetRulePrioritiesInput(matchedResAndSDKLRsBySettings []resAndSDKListenerRulePair, unmatchedSDKLRs []ListenerRuleWithTags) *elbv2sdk.SetRulePrioritiesInput {
201+
var rulePriorities []elbv2types.RulePriorityPair
202+
var lastAvailablePriority int32 = 50000
203+
var sdkLRs []ListenerRuleWithTags
204+
205+
// Sort the unmatched existing SDK rules based on their priority to be pushed down in same order
206+
sort.Slice(unmatchedSDKLRs, func(i, j int) bool {
207+
priorityI, _ := strconv.Atoi(awssdk.ToString(unmatchedSDKLRs[i].ListenerRule.Priority))
208+
priorityJ, _ := strconv.Atoi(awssdk.ToString(unmatchedSDKLRs[j].ListenerRule.Priority))
209+
return priorityI < priorityJ
210+
})
211+
// Push down all the unmatched existing SDK rules on load balancer so that updated rules can take their place
212+
for _, sdkLR := range slices.Backward(unmatchedSDKLRs) {
213+
sdkLR.ListenerRule.Priority = awssdk.String(strconv.Itoa(int(lastAvailablePriority)))
214+
sdkLRs = append(sdkLRs, sdkLR)
215+
lastAvailablePriority--
216+
}
217+
//Re-Prioritize matched rules by settings
218+
for _, resAndSDKLR := range matchedResAndSDKLRsBySettings {
219+
resAndSDKLR.sdkLR.ListenerRule.Priority = awssdk.String(strconv.Itoa(int(resAndSDKLR.resLR.Spec.Priority)))
220+
sdkLRs = append(sdkLRs, resAndSDKLR.sdkLR)
221+
}
222+
for _, sdkLR := range sdkLRs {
223+
p, _ := strconv.ParseInt(awssdk.ToString(sdkLR.ListenerRule.Priority), 10, 32)
224+
rulePriorityPair := elbv2types.RulePriorityPair{
225+
RuleArn: sdkLR.ListenerRule.RuleArn,
226+
Priority: awssdk.Int32(int32(p)),
227+
}
228+
rulePriorities = append(rulePriorities, rulePriorityPair)
229+
}
230+
sdkObj := &elbv2sdk.SetRulePrioritiesInput{
231+
RulePriorities: rulePriorities,
232+
}
233+
return sdkObj
234+
}
190235
func buildResListenerRuleStatus(sdkLR ListenerRuleWithTags) elbv2model.ListenerRuleStatus {
191236
return elbv2model.ListenerRuleStatus{
192237
RuleARN: awssdk.ToString(sdkLR.ListenerRule.RuleArn),

0 commit comments

Comments
 (0)