Skip to content

Commit f064b32

Browse files
ivanmatmatioktalz
authored andcommitted
MINOR: add SNI map for TLS route with passthrough
1 parent 71c0547 commit f064b32

File tree

14 files changed

+253
-53
lines changed

14 files changed

+253
-53
lines changed

.aspell.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,4 @@ allowed:
7676
- darwin
7777
- performant
7878
- dockerhub
79+
- SNI

hug/haproxy/api/tcp-requests.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818

1919
parser "github.com/haproxytech/client-native/v6/config-parser"
2020
"github.com/haproxytech/client-native/v6/models"
21-
"github.com/haproxytech/kubernetes-controller/k8s/gate/logging"
21+
"github.com/haproxytech/haproxy-unified-gateway/k8s/gate/logging"
2222
)
2323

2424
func (c *clientNative) TCPRequestDeleteAll(parentType parser.Section, name string) error {

k8s/gate/haproxy/backends.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,6 @@ func (b *HaproxyConfMgrImpl) upsertTLSRouteBackends(routeKey k8stypes.Namespaced
291291
b.backendsImpactedInCycle.Unreferenced[unreferencedBeName] = struct{}{}
292292
}
293293
return errs.Result()
294-
295294
}
296295

297296
func (b *HaproxyConfMgrImpl) processTLSRoutes() error {
@@ -382,7 +381,7 @@ func (b *HaproxyConfMgrImpl) addImpactedLTSBackendUpserted(backendName string, r
382381
impactedBe := BackendImpactedInCycle{
383382
Name: backendName,
384383
HTTPRouteKey: routeKey,
385-
//BackendRef: tlsBackendRef,
384+
// BackendRef: tlsBackendRef,
386385
}
387386

388387
if _, ok := b.backendsImpactedInCycle.Upserted[backendName]; !ok {
@@ -607,6 +606,7 @@ func DeepCopyBackend(original *models.Backend) (*models.Backend, error) {
607606
return &copied, nil
608607
}
609608

609+
//revive:disable:function-length
610610
func (b *HaproxyConfMgrImpl) processBackendsModifiedInCycle() error {
611611
var errs utils.Errors
612612
// UPSERTED

k8s/gate/haproxy/frontends.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -255,12 +255,12 @@ func (b *HaproxyConfMgrImpl) newFrontend(params newFrontendParams) (*models.Fron
255255
CondTest: "!{ req_ssl_hello_type 1 }",
256256
},
257257
{
258-
//tcp-request inspect-delay 50000
258+
// tcp-request inspect-delay 50000
259259
Type: "inspect-delay",
260260
Timeout: utils.PtrInt64(50000),
261261
},
262262
{
263-
//tcp-request content set-var(sess.sni) req_ssl_sni
263+
// tcp-request content set-var(sess.sni) req_ssl_sni
264264
Type: "content",
265265
Action: "set-var",
266266
VarName: "sni",

k8s/gate/haproxy/routes-maps.go

Lines changed: 73 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,78 @@ func (e *ErrMapRuntimeUpdate) Error() string {
4040
}
4141

4242
func (b *RouteMgrImpl) fillMaps() {
43+
b.fillMapsForHTTPRoutes()
44+
b.fillMapsForTLSRoutes()
45+
}
46+
47+
func (b *RouteMgrImpl) fillMapsForTLSRoutes() {
4348
var errs utils.Errors
4449
controllerStore := b.topManager.controllerStore
4550
mapsStorage := b.topManager.params.mapsStorage
51+
// Managed TLSRoutes => if there is a old resource, clean the state before update
52+
for routeKey, route := range controllerStore.GateTree.TLSRoutes {
53+
if route.TreeStatus.OldTreeResource == nil {
54+
continue
55+
}
56+
route = route.TreeStatus.OldTreeResource
57+
for _, listeners := range route.Listeners.Iterate {
58+
for _, listener := range listeners {
59+
frontendName, err := b.topManager.getFrontendName(listener.Owner, listener.K8sResource)
60+
if err != nil {
61+
b.topManager.logger.LogAttrs(context.Background(), slog.LevelError, "Failed to get frontend name",
62+
logging.LogAttrError(err),
63+
)
64+
}
65+
66+
pathSNIMap := mapsStorage.MapPath(frontendName, storage.SNI_MAP)
67+
mapSNIMap := mapsStorage.GetMapData(pathSNIMap)
68+
err = b.onDeletedTLSRoute(routeKey, route, mapSNIMap)
69+
// errs.Add(err)
70+
_ = err // TODO ignore error for now
71+
}
72+
}
73+
}
74+
// Managed TLSRoutes => Create / update/ delete backends
75+
for routeKey, route := range controllerStore.GateTree.TLSRoutes {
76+
for _, listeners := range route.Listeners.Iterate {
77+
for _, listener := range listeners {
78+
frontendName, err := b.topManager.getFrontendName(listener.Owner, listener.K8sResource)
79+
if err != nil {
80+
b.topManager.logger.LogAttrs(context.Background(), slog.LevelError, "Failed to get frontend name",
81+
logging.LogAttrError(err),
82+
)
83+
}
84+
routesHosnames := utils.ConvertSliceWithFunc(route.K8sResource.Spec.Hostnames, utils.ConvertV1Alpha2HostnameToString)
85+
listenerHostname := (*string)(listener.K8sResource.Hostname)
86+
acceptedHostnamesForRoute := utils.MatchTLSHostnames(listenerHostname, routesHosnames)
87+
pathSNIMap := mapsStorage.MapPath(frontendName, storage.SNI_MAP)
88+
mapSNIMap := mapsStorage.GetMapData(pathSNIMap)
89+
90+
switch route.TreeStatus.Status {
91+
case store.StatusUnchanged:
92+
continue
93+
case store.StatusUpserted:
94+
err := b.onUpsertedTLSRoute(routeKey, route, mapSNIMap, acceptedHostnamesForRoute)
95+
errs.Add(err)
96+
case store.StatusDeleted:
97+
err := b.onDeletedTLSRoute(routeKey, route, mapSNIMap)
98+
errs.Add(err)
99+
}
100+
}
101+
}
102+
}
103+
104+
if len(errs) > 0 {
105+
b.topManager.logger.LogAttrs(context.Background(), slog.LevelError, "Failed to fill maps for TLS routes",
106+
logging.LogAttrError(errs.Result()),
107+
)
108+
}
109+
}
46110

111+
func (b *RouteMgrImpl) fillMapsForHTTPRoutes() {
112+
var errs utils.Errors
113+
controllerStore := b.topManager.controllerStore
114+
mapsStorage := b.topManager.params.mapsStorage
47115
// Managed HTTPRoutes => if there is a old resource, clean the state before update
48116
for routeKey, route := range controllerStore.GateTree.HTTPRoutes {
49117
if route.TreeStatus.OldTreeResource == nil {
@@ -107,24 +175,11 @@ func (b *RouteMgrImpl) fillMaps() {
107175
}
108176
}
109177
}
110-
111-
// Cleanup Backends that are not referenced anymore TODO
112-
// if err := b.cleanupUnreferencedBackends(); err != nil {
113-
// b.logger.LogAttrs(context.Background(), slog.LevelError, "Failed to cleanup unreferenced backends",
114-
// logging.LogAttrError(err),
115-
// )
116-
// errs.Add(err)
117-
// }
118-
119-
// // Now we have the list of upserted + delete BE with the correct list of routes pointing to them
120-
// if err := b.processBackendsModifiedInCycle(); err != nil {
121-
// b.logger.LogAttrs(context.Background(), slog.LevelError, "Failed to process backends modified in cycle",
122-
// logging.LogAttrError(err),
123-
// )
124-
// errs.Add(err)
125-
// }
126-
127-
// return errs.Result()
178+
if len(errs) > 0 {
179+
b.topManager.logger.LogAttrs(context.Background(), slog.LevelError, "Failed to fill maps for HTTP routes",
180+
logging.LogAttrError(errs.Result()),
181+
)
182+
}
128183
}
129184

130185
func (b *RouteMgrImpl) writeMaps() {

k8s/gate/haproxy/routes.go

Lines changed: 106 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,106 @@ func (b *RouteMgrImpl) onUpsertedHTTPRoute(routeKey k8stypes.NamespacedName, rou
6161
return b.onInvalidHTTPRouteUpserted(routeKey, route, mapExact, mapPrefix, mapRegex, mapDomainWPathExact)
6262
}
6363

64+
func (b *RouteMgrImpl) onUpsertedTLSRoute(routeKey k8stypes.NamespacedName, route *tree.TLSRoute,
65+
sni *maps.MapData, acceptedHostnamesForRoute []string,
66+
) error {
67+
if route.Valid {
68+
return b.onValidTLSRouteUpserted(routeKey, route, sni, acceptedHostnamesForRoute)
69+
}
70+
return b.onInvalidTLSRouteUpserted(routeKey, route, sni)
71+
}
72+
73+
func (b *RouteMgrImpl) onValidTLSRouteUpserted(_ k8stypes.NamespacedName,
74+
tlsRoute *tree.TLSRoute, mapSNI *maps.MapData, acceptedHostnamesForRoute []string) error {
75+
76+
for _, tlsRouteRule := range tlsRoute.Rules {
77+
// if !rule.Valid {
78+
// find the old rule in route.TreeStatus.OldTreeResource.Rules, name is optional
79+
// TODO
80+
// }
81+
var routeValue string
82+
var backendNames []string
83+
var backendweights []int32
84+
hostnamesInserted := map[string]struct{}{}
85+
for _, backend := range tlsRouteRule.K8sResource.BackendRefs {
86+
checkResult, ok := tlsRouteRule.CheckBackendRef.Get(backend.BackendObjectReference)
87+
if !ok || !checkResult.Valid {
88+
b.topManager.logger.LogAttrs(context.Background(), slog.LevelDebug, "Processing TLSRoute [map update] - backend not valid",
89+
logging.LogAttrBackendName(string(backend.Name)),
90+
)
91+
continue
92+
}
93+
94+
// backend := rule.K8sResource.BackendRefs[index]
95+
svckey := k8stypes.NamespacedName{
96+
Name: string(backend.Name),
97+
}
98+
if backend.Namespace == nil {
99+
svckey.Namespace = tlsRoute.K8sResource.Namespace
100+
} else {
101+
svckey.Namespace = string(*backend.Namespace)
102+
}
103+
svcPort := int32(0)
104+
if backend.Port != nil {
105+
svcPort = int32(*backend.Port)
106+
}
107+
108+
backendName, err := b.topManager.getBackendName(svckey, int32(svcPort), "")
109+
if err != nil {
110+
b.topManager.logger.LogAttrs(context.Background(), slog.LevelError, "Processing TLSRoute [map update]",
111+
logging.LogAttrError(err),
112+
)
113+
continue
114+
}
115+
backendNames = append(backendNames, backendName)
116+
weight := int32(0)
117+
if backend.Weight != nil {
118+
weight = *backend.Weight
119+
}
120+
backendweights = append(backendweights, weight)
121+
}
122+
if len(backendNames) == 1 {
123+
routeValue = backendNames[0]
124+
} else {
125+
// a: algo, s: suffix, l: list of backend (format depends on algo)
126+
// /wr_a70_b20_c10 {"a":"wr","l":"a:70,b:20,c:10"}
127+
routeValue = `{"a":"wr","l":"`
128+
for i, backendName := range backendNames {
129+
if i > 0 {
130+
routeValue += ","
131+
}
132+
routeValue += fmt.Sprintf("%s:%d", backendName, backendweights[i])
133+
}
134+
routeValue += `"}`
135+
}
136+
137+
for _, hostname := range acceptedHostnamesForRoute {
138+
if tlsRouteRule.Valid {
139+
mapSNI.AddData(string(hostname), routeValue)
140+
hostnamesInserted[string(hostname)] = struct{}{}
141+
} else if _, ok := hostnamesInserted[string(hostname)]; !ok {
142+
mapSNI.DeleteData(string(hostname))
143+
}
144+
}
145+
}
146+
return nil
147+
}
148+
149+
func (RouteMgrImpl) onInvalidTLSRouteUpserted(_ k8stypes.NamespacedName, _ *tree.TLSRoute, _ *maps.MapData) error {
150+
// TODO we might need to remove it from the maps
151+
return nil
152+
}
153+
154+
func (RouteMgrImpl) onDeletedTLSRoute(_ k8stypes.NamespacedName, route *tree.TLSRoute,
155+
mapSNI *maps.MapData,
156+
) error {
157+
hostnames := route.K8sResource.Spec.Hostnames
158+
for _, hostname := range hostnames {
159+
mapSNI.DeleteData(string(hostname))
160+
}
161+
return nil
162+
}
163+
64164
func (RouteMgrImpl) onDeletedHTTPRoute(_ k8stypes.NamespacedName, route *tree.HTTPRoute,
65165
// func (b *RouteMgrImpl) onDeletedHTTPRoute(routeKey k8stypes.NamespacedName, route *tree.HTTPRoute,
66166
mapExact, mapPrefix, mapRegex, mapDomainWPathExact *maps.MapData,
@@ -119,7 +219,7 @@ func (b *RouteMgrImpl) onValidHTTPRouteUpserted(_ k8stypes.NamespacedName, route
119219
// TODO
120220
// }
121221
var routeValue string
122-
var backendNames []string
222+
var backendNamesForRule []string
123223
var backendweights []int32
124224
for index, backend := range rule.K8sResource.BackendRefs {
125225
checkResult, ok := rule.CheckBackendRef.Get(backend.BackendObjectReference)
@@ -151,20 +251,20 @@ func (b *RouteMgrImpl) onValidHTTPRouteUpserted(_ k8stypes.NamespacedName, route
151251
)
152252
continue
153253
}
154-
backendNames = append(backendNames, backendName)
254+
backendNamesForRule = append(backendNamesForRule, backendName)
155255
weight := int32(0)
156256
if backend.Weight != nil {
157257
weight = *backend.Weight
158258
}
159259
backendweights = append(backendweights, weight)
160260
}
161-
if len(backendNames) == 1 {
162-
routeValue = backendNames[0]
163-
} else {
261+
if len(backendNamesForRule) == 1 {
262+
routeValue = backendNamesForRule[0]
263+
} else if len(backendNamesForRule) > 1 {
164264
// a: algo, s: suffix, l: list of backend (format depends on algo)
165265
// /wr_a70_b20_c10 {"a":"wr","l":"a:70,b:20,c:10"}
166266
routeValue = `{"a":"wr","l":"`
167-
for i, backendName := range backendNames {
267+
for i, backendName := range backendNamesForRule {
168268
if i > 0 {
169269
routeValue += ","
170270
}

k8s/gate/haproxy/servers.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,8 @@ func (b *HaproxyConfMgrImpl) getSvcEndpoints(ctx context.Context, svcKey client.
313313

314314
func (b *HaproxyConfMgrImpl) backendsByServiceForHTTPRoute(routeOwners map[client.ObjectKey]int64,
315315
beName string,
316-
servicesByBackend map[client.ObjectKey]map[BackendPort]struct{}) {
316+
servicesByBackend map[client.ObjectKey]map[BackendPort]struct{},
317+
) {
317318
for ownerRouteKey := range routeOwners {
318319
// Find the route in controllerStore GateTree
319320
treeHTTPRoute, ok := b.controllerStore.GateTree.HTTPRoutes[ownerRouteKey]
@@ -358,7 +359,8 @@ func (b *HaproxyConfMgrImpl) backendsByServiceForHTTPRoute(routeOwners map[clien
358359

359360
func (b *HaproxyConfMgrImpl) backendsByServiceForTLSRoute(routeOwners map[client.ObjectKey]int64,
360361
beName string,
361-
servicesByBackend map[client.ObjectKey]map[BackendPort]struct{}) {
362+
servicesByBackend map[client.ObjectKey]map[BackendPort]struct{},
363+
) {
362364
for ownerRouteKey := range routeOwners {
363365
// Find the route in controllerStore GateTree
364366
treeTLSRoute, ok := b.controllerStore.GateTree.TLSRoutes[ownerRouteKey]

k8s/gate/store/cluster-store.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import (
2727
"k8s.io/apimachinery/pkg/types"
2828
"sigs.k8s.io/controller-runtime/pkg/client"
2929
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
30-
"sigs.k8s.io/gateway-api/apis/v1alpha2"
3130
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
3231
)
3332

@@ -78,7 +77,7 @@ func NewClusterStoreUpdaterImpl(
7877
extractGVK(&gatewayv1.GatewayClass{}): newObjectStoreImpl(clusterStore.GatewayClasses, clusterStore.Updates.GatewayClasses, logger),
7978
extractGVK(&gatewayv1.Gateway{}): newObjectStoreImpl(clusterStore.Gateways, clusterStore.Updates.Gateways, logger),
8079
extractGVK(&gatewayv1.HTTPRoute{}): newObjectStoreImpl(clusterStore.HTTPRoutes, clusterStore.Updates.HTTPRoutes, logger),
81-
extractGVK(&v1alpha2.TLSRoute{}): newObjectStoreImpl(clusterStore.TLSRoutes, clusterStore.Updates.TLSRoutes, logger),
80+
extractGVK(&gatewayv1alpha2.TLSRoute{}): newObjectStoreImpl(clusterStore.TLSRoutes, clusterStore.Updates.TLSRoutes, logger),
8281
extractGVK(&v1.Service{}): newObjectStoreImpl(clusterStore.Services, clusterStore.Updates.Services, logger),
8382
extractGVK(&v1.Namespace{}): newObjectStoreImpl(clusterStore.Namespaces, clusterStore.Updates.Namespaces, logger),
8483
extractGVK(&v1.Secret{}): newObjectStoreImpl(clusterStore.Secrets, clusterStore.Updates.Secrets, logger),

k8s/gate/store/cluster-updates.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"k8s.io/apimachinery/pkg/types"
2323
"sigs.k8s.io/controller-runtime/pkg/client"
2424
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
25-
"sigs.k8s.io/gateway-api/apis/v1alpha2"
2625
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
2726
)
2827

@@ -70,7 +69,7 @@ func NewClusterUpdates() ClusterUpdates {
7069
GatewayClasses: make(map[types.NamespacedName]Update[*gatewayv1.GatewayClass]),
7170
Gateways: make(map[types.NamespacedName]Update[*gatewayv1.Gateway]),
7271
HTTPRoutes: make(map[types.NamespacedName]Update[*gatewayv1.HTTPRoute]),
73-
TLSRoutes: make(map[types.NamespacedName]Update[*v1alpha2.TLSRoute]),
72+
TLSRoutes: make(map[types.NamespacedName]Update[*gatewayv1alpha2.TLSRoute]),
7473
Services: make(map[types.NamespacedName]Update[*v1.Service]),
7574
Namespaces: make(map[types.NamespacedName]Update[*v1.Namespace]),
7675
Secrets: make(map[types.NamespacedName]Update[*v1.Secret]),

k8s/gate/tree/TLSRoute.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ func (r *TLSRoute) checkParentRefs(controllerStore ControllerStore) {
166166
r.CheckParentRefs = checkParentsRefs
167167
}
168168

169+
//revive:disable:function-length
169170
func (r *TLSRoute) checkParentRef(parentRef gatewayv1.ParentReference, controllerStore ControllerStore) checkParentRefResult {
170171
if !isParentRefGroupKindSupported(parentRef, controllerStore.ExtractGVK) {
171172
return checkParentRefResult{

0 commit comments

Comments
 (0)