feat: add EKS Hybrid Nodes support (#4336)

* fix hybrid-pods eni resolver

* fix caching and add tests

* use const for hybridNetworkInterfaceID

Author: sanbyk

pkg/k8s/node_utils.go

@@ -8,6 +8,7 @@ import (
 )
 
 var awsInstanceIDRegex = regexp.MustCompile("^i-[^/]*$")
+var eksHybridIDRegex = regexp.MustCompile("^mi-[^/]*$")
 
 // GetNodeCondition will get pointer to Node's existing condition.
 // returns nil if no matching condition found.
@@ -26,6 +27,18 @@ func ExtractNodeInstanceID(node *corev1.Node) (string, error) {
 		return "", errors.Errorf("providerID is not specified for node: %s", node.Name)
 	}
 
+	// Check if this is a hybrid node
+	if strings.HasPrefix(providerID, "eks-hybrid://") {
+		providerIDParts := strings.Split(providerID, "/")
+		hybridID := providerIDParts[len(providerIDParts)-1]
+		if !eksHybridIDRegex.MatchString(hybridID) {
+			return "", errors.Errorf("providerID %s is invalid for EKS hybrid instances, node: %s", providerID, node.Name)
+		}
+		// Return a special prefix to identify hybrid nodes
+		return "hybrid-" + hybridID, nil
+	}
+
+	// Handle EC2 instances (existing logic)
 	providerIDParts := strings.Split(providerID, "/")
 	instanceID := providerIDParts[len(providerIDParts)-1]
 	if !awsInstanceIDRegex.MatchString(instanceID) {

pkg/k8s/node_utils_test.go

@@ -102,6 +102,20 @@ func TestExtractNodeInstanceID(t *testing.T) {
 			},
 			want: "i-abcdefg0",
 		},
+		{
+			name: "node by EKS Hybrid",
+			args: args{
+				node: &corev1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "hybrid-node-name",
+					},
+					Spec: corev1.NodeSpec{
+						ProviderID: "eks-hybrid:///ap-northeast-1/eks-test/mi-01c49bb1234567890",
+					},
+				},
+			},
+			want: "hybrid-mi-01c49bb1234567890",
+		},
 		{
 			name: "node by EKS Fargate",
 			args: args{

pkg/networking/networking_manager.go

@@ -161,6 +161,10 @@ func (m *defaultNetworkingManager) computeIngressPermissionsPerSGWithPodEndpoint
 
 	podsBySG := make(map[string][]k8s.PodInfo)
 	for podKey, eniInfo := range eniInfoByPodKey {
+		// Handle hybrid pods specially - they don't have security groups
+		if eniInfo.NetworkInterfaceID == hybridNetworkInterfaceID {
+			continue
+		}
 		sgID, err := m.resolveEndpointSGForENI(ctx, eniInfo)
 		if err != nil {
 			return nil, err

pkg/networking/node_info_provider.go

@@ -46,13 +46,22 @@ func (p *defaultNodeInfoProvider) FetchNodeInstances(ctx context.Context, nodes
 	}
 	nodeKeysByInstanceID := make(map[string][]types.NamespacedName, len(nodes))
 	for _, node := range nodes {
+		if node.Labels["eks.amazonaws.com/compute-type"] == "hybrid" {
+			continue
+		}
 		instanceID, err := k8s.ExtractNodeInstanceID(node)
 		if err != nil {
 			return nil, err
 		}
 		nodeKey := k8s.NamespacedName(node)
 		nodeKeysByInstanceID[instanceID] = append(nodeKeysByInstanceID[instanceID], nodeKey)
 	}
+
+	// If no EC2 instances to fetch, return empty result
+	if len(nodeKeysByInstanceID) == 0 {
+		return make(map[types.NamespacedName]*ec2types.Instance), nil
+	}
+
 	instanceIDs := sets.StringKeySet(nodeKeysByInstanceID).List()
 	req := &ec2sdk.DescribeInstancesInput{
 		InstanceIds: instanceIDs,

pkg/networking/pod_eni_info_resolver.go

@@ -28,6 +28,7 @@ const (
 
 	labelEKSComputeType       = "eks.amazonaws.com/compute-type"
 	labelSageMakerComputeType = "sagemaker.amazonaws.com/compute-type"
+	hybridNetworkInterfaceID  = "hybrid-no-eni"
 )
 
 // PodENIInfoResolver is responsible for resolve the AWS VPC ENI that supports pod network.
@@ -176,6 +177,18 @@ func (r *defaultPodENIInfoResolver) resolvePodsViaCascadedLookup(ctx context.Con
 			}
 		}
 	}
+	// Hybrid pods don't have ENIs - they're connected via Direct Connect
+	// We'll handle them specially in the networking manager
+	if len(podsByComputeType.hybridPods) > 0 {
+		// Return empty ENI info for hybrid pods - they'll be handled specially
+		for _, pod := range podsByComputeType.hybridPods {
+			eniInfoByPodKey[pod.Key] = ENIInfo{
+				// Use a special identifier to mark this as a hybrid pod
+				NetworkInterfaceID: hybridNetworkInterfaceID,
+				SecurityGroups:     []string{},
+			}
+		}
+	}
 	return eniInfoByPodKey, nil
 }
 
@@ -401,14 +414,15 @@ func (r *defaultPodENIInfoResolver) isPodSupportedByNodeENI(pod k8s.PodInfo, nod
 	return false
 }
 
-// PodsByComputeType groups pods based on their compute type (EC2, Fargate, SageMaker HyperPod)
+// PodsByComputeType groups pods based on their compute type (EC2, Fargate, SageMaker HyperPod, Hybrid)
 type PodsByComputeType struct {
 	ec2Pods               []k8s.PodInfo
 	fargatePods           []k8s.PodInfo
 	sageMakerHyperPodPods []k8s.PodInfo
+	hybridPods            []k8s.PodInfo
 }
 
-// classifyPodsByComputeType classifies in to ec2, fargate and sagemaker-hyperpod groups
+// classifyPodsByComputeType classifies in to ec2, fargate, sagemaker-hyperpod and hybrid groups
 func (r *defaultPodENIInfoResolver) classifyPodsByComputeType(ctx context.Context, pods []k8s.PodInfo) (PodsByComputeType, error) {
 	var podsByComputeType PodsByComputeType
 	nodeNameByComputeType := make(map[string]string)
@@ -418,9 +432,12 @@ func (r *defaultPodENIInfoResolver) classifyPodsByComputeType(ctx context.Contex
 				podsByComputeType.fargatePods = append(podsByComputeType.fargatePods, pod)
 			} else if nodeNameByComputeType[pod.NodeName] == "sagemaker-hyperpod" {
 				podsByComputeType.sageMakerHyperPodPods = append(podsByComputeType.sageMakerHyperPodPods, pod)
+			} else if nodeNameByComputeType[pod.NodeName] == "hybrid" {
+				podsByComputeType.hybridPods = append(podsByComputeType.hybridPods, pod)
 			} else {
 				podsByComputeType.ec2Pods = append(podsByComputeType.ec2Pods, pod)
 			}
+			continue // Skip the rest of the loop iteration since we already processed this pod
 		}
 
 		nodeKey := types.NamespacedName{Name: pod.NodeName}
@@ -434,6 +451,9 @@ func (r *defaultPodENIInfoResolver) classifyPodsByComputeType(ctx context.Contex
 		} else if node.Labels[labelSageMakerComputeType] == "hyperpod" {
 			podsByComputeType.sageMakerHyperPodPods = append(podsByComputeType.sageMakerHyperPodPods, pod)
 			nodeNameByComputeType[pod.NodeName] = "sagemaker-hyperpod"
+		} else if node.Labels[labelEKSComputeType] == "hybrid" {
+			podsByComputeType.hybridPods = append(podsByComputeType.hybridPods, pod)
+			nodeNameByComputeType[pod.NodeName] = "hybrid"
 		} else {
 			podsByComputeType.ec2Pods = append(podsByComputeType.ec2Pods, pod)
 			nodeNameByComputeType[pod.NodeName] = "ec2"