From 4981db4500c2e1a1b4edbbd68a7a75ff3c8b4d2f Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Mon, 13 Oct 2025 10:20:35 +0000 Subject: [PATCH] minikube: Add integration test for metallb addon Signed-off-by: Kartik Joshi --- test/integration/addons_test.go | 315 ++++++++++++++++++++++++++++++-- 1 file changed, 303 insertions(+), 12 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 6c24a01d5d0b..b40d83353051 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -25,6 +25,7 @@ import ( "encoding/json" "errors" "fmt" + "net" "net/http" "net/url" "os" @@ -141,18 +142,19 @@ func TestAddons(t *testing.T) { t.Run("parallel", func(t *testing.T) { tests := []TestCase{ {"Registry", validateRegistryAddon}, - {"RegistryCreds", validateRegistryCredsAddon}, - {"Ingress", validateIngressAddon}, - {"InspektorGadget", validateInspektorGadgetAddon}, - {"MetricsServer", validateMetricsServerAddon}, - {"Olm", validateOlmAddon}, - {"CSI", validateCSIDriverAndSnapshots}, - {"Headlamp", validateHeadlampAddon}, - {"CloudSpanner", validateCloudSpannerAddon}, - {"LocalPath", validateLocalPathAddon}, - {"NvidiaDevicePlugin", validateNvidiaDevicePlugin}, - {"Yakd", validateYakdAddon}, - {"AmdGpuDevicePlugin", validateAmdGpuDevicePlugin}, + {"RegistryCreds", validateRegistryCredsAddon}, + {"Ingress", validateIngressAddon}, + {"InspektorGadget", validateInspektorGadgetAddon}, + {"MetricsServer", validateMetricsServerAddon}, + {"Olm", validateOlmAddon}, + {"CSI", validateCSIDriverAndSnapshots}, + {"Headlamp", validateHeadlampAddon}, + {"CloudSpanner", validateCloudSpannerAddon}, + {"LocalPath", validateLocalPathAddon}, + {"NvidiaDevicePlugin", validateNvidiaDevicePlugin}, + {"Yakd", validateYakdAddon}, + {"AmdGpuDevicePlugin", validateAmdGpuDevicePlugin}, + {"MetalLB", validateMetalLBAddon}, } for _, tc := range tests { @@ -189,6 +191,295 @@ func TestAddons(t *testing.T) { }) } +// --- MetalLB helpers --------------------------------------------------------- + +// hasUsableMetalLBPool checks whether MetalLB has at least one non-empty address +// range configured (CRD or legacy ConfigMap). +func hasUsableMetalLBPool(ctx context.Context, t *testing.T, profile string, useCRD bool) bool { + if useCRD { + // If any item has a non-empty spec.addresses[0], we consider it usable. + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, + "-n", "metallb-system", "get", "ipaddresspools", + "-o", "jsonpath={range .items[*]}{.spec.addresses[0]}{'\\n'}{end}")) + if err != nil { + return false + } + for _, line := range strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n") { + if s := strings.TrimSpace(line); s != "" && s != "[]" && !strings.EqualFold(s, "") { + t.Logf("MetalLB CRD pool present: %q", s) + return true + } + } + return false + } + // Legacy CM: parse the blob for a non-empty addresses list or at least one dash entry. + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, + "-n", "metallb-system", "get", "cm", "config", "-o", "jsonpath={.data.config}")) + if err != nil { + return false + } + cfg := rr.Stdout.String() + // Look for: addresses: ["A-B"] OR addresses:\n - A-B + rxNonEmptyInline := regexp.MustCompile(`(?m)addresses:\s*\[\s*("[^"\]]+"\s*(,\s*"[^"\]]+")*)\s*\]`) + rxDash := regexp.MustCompile(`(?m)^\s*-\s*[0-9a-fA-F:.]+(?:\s*-\s*[0-9a-fA-F:.]+)?\s*$`) + if rxNonEmptyInline.FindString(cfg) != "" || rxDash.FindString(cfg) != "" { + t.Logf("MetalLB CM appears configured") + return true + } + return false +} + +// ensureMetalLBConfigured detects CRD vs legacy ConfigMap mode and applies +// an address pool carved from the tail of the node's IPv4 subnet. This lets +// MetalLB work on Hyper-V External switches where the node lives on the LAN. +func ensureMetalLBConfigured(ctx context.Context, t *testing.T, profile string) { + t.Logf("configuring metallb address pool") + + // Detect CRD mode + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, + "api-resources", "--api-group=metallb.io", "-o", "name")) + useCRD := (err == nil) && strings.Contains(rr.Stdout.String(), "ipaddresspools.metallb.io") + + // Already configured? + // Already configured with a usable (non-empty) pool? + if hasUsableMetalLBPool(ctx, t, profile, useCRD) { + return + } + + // Work out the node's IPv4 CIDR (eth0 inside minikube) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, + "ssh", "ip -4 -o addr show dev eth0 | awk '{print $4}'")) + if err != nil { + t.Fatalf("failed to get node CIDR: %v", err) + } + cidr := strings.TrimSpace(rr.Stdout.String()) + _, ipNet, perr := net.ParseCIDR(cidr) + if perr != nil { + t.Fatalf("parse CIDR %q: %v", cidr, perr) + } + + // Tail of subnet: broadcast-32 .. broadcast-4 + dec := func(ip net.IP, n int) net.IP { + out := append(net.IP(nil), ip...) + for ; n > 0; n-- { + for j := len(out) - 1; j >= 0; j-- { + if out[j] > 0 { + out[j]-- + break + } + out[j] = 255 + } + } + return out + } + network := ipNet.IP.Mask(ipNet.Mask) + bcast := make(net.IP, len(network)) + for i := range network { + bcast[i] = network[i] | ^ipNet.Mask[i] + } + start, end := dec(bcast, 32), dec(bcast, 4) + + if useCRD { + manifest := fmt.Sprintf(` +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: itest-pool + namespace: metallb-system +spec: + addresses: ["%s-%s"] +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: itest-l2 + namespace: metallb-system +`, start.String(), end.String()) + cmd := exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", "-") + cmd.Stdin = bytes.NewBufferString(manifest) + if _, err := Run(t, cmd); err != nil { + t.Fatalf("apply MetalLB CRDs: %v", err) + } + } else { + manifest := fmt.Sprintf(` +apiVersion: v1 +kind: ConfigMap +metadata: + name: config + namespace: metallb-system +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: ["%s-%s"] +`, start.String(), end.String()) + cmd := exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", "-") + cmd.Stdin = bytes.NewBufferString(manifest) + if _, err := Run(t, cmd); err != nil { + t.Fatalf("apply MetalLB CM: %v", err) + } + } +} + +// validateMetalLBAddon tests MetalLB by exposing a tiny HTTP pod via a LoadBalancer +// and verifying it receives an external IP and is reachable from the host. +func validateMetalLBAddon(ctx context.Context, t *testing.T, profile string) { + // MetalLB exercises host<->cluster L2/L3 path; skip where that’s not viable. + t.Logf("Running MetalLB e2e") + + if NoneDriver() { + t.Skipf("skipping: metallb not supported on 'none' driver in this test") + } + if NeedsPortForward() { + t.Skipf("skipping metallb test: environment requires port-forwarding") + } + defer disableAddon(t, "metallb", profile) + defer PostMortemLogs(t, profile) + + client, err := kapi.Client(profile) + if err != nil { + t.Fatalf("failed to get Kubernetes client: %v", err) + } + + // Enable MetalLB + if rr, err := Run(t, exec.CommandContext(ctx, Target(), "--alsologtostderr", "-v=1", "addons", "enable", "metallb", "-p", profile)); err != nil { + t.Fatalf("failed to enable metallb addon: args %q : %v", rr.Command(), err) + } + + // Auto-configure an IP pool that matches the node's LAN subnet (CRD or legacy CM). + ensureMetalLBConfigured(ctx, t, profile) + + // Wait for controller to stabilize. Name differs across versions: + // try "controller" first, fall back to "metallb-controller". + start := time.Now() + waitController := func(name string) error { + return kapi.WaitForDeploymentToStabilize(client, "metallb-system", name, Minutes(6)) + } + if err := waitController("controller"); err != nil { + t.Logf("metallb deployment 'controller' not ready (%v); trying 'metallb-controller'", err) + if err2 := waitController("metallb-controller"); err2 != nil { + t.Fatalf("metallb controller failed to stabilize: %v / %v", err, err2) + } + } + // Also wait for the speaker DaemonSet to be ready to avoid LB IP allocation races. + if rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "-n", "metallb-system", + "rollout", "status", "ds/speaker", "--timeout=180s")); err != nil { + t.Fatalf("failed waiting for metallb speaker ds to be ready: args %q : %v", rr.Command(), err) + } + t.Logf("metallb controller stabilized in %s", time.Since(start)) + + // Create a tiny HTTP server pod (busybox httpd) with a unique label to avoid collisions. + // We avoid YAML here to keep the patch small and independent of other tests' resources. + if rr, err := Run(t, exec.CommandContext( + ctx, "kubectl", "--context", profile, + "run", "mlb-http", + "--image=gcr.io/k8s-minikube/busybox", + "--labels", "app=mlb-http", + "--restart=Never", + "--command", "--", "sh", "-c", + // create simple index and serve /www + "set -eu; mkdir -p /www; echo ok >/www/index.html; exec httpd -f -p 80 -h /www")); err != nil { + t.Fatalf("failed to create mlb-http pod: args %q : %v", rr.Command(), err) + } + + if _, err := PodWait(ctx, t, profile, "default", "app=mlb-http", Minutes(6)); err != nil { + t.Fatalf("failed waiting for mlb-http pod: %v", err) + } + + // Expose it as a LoadBalancer service. + if rr, err := Run(t, exec.CommandContext( + ctx, "kubectl", "--context", profile, + "expose", "pod", "mlb-http", + "--name", "mlb-http-lb", + "--type", "LoadBalancer", + "--port", "80", + "--target-port", "80")); err != nil { + t.Fatalf("failed to expose mlb-http-lb service: args %q : %v", rr.Command(), err) + } + t.Cleanup(func() { + // best-effort cleanup + Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "svc", "mlb-http-lb", "--now")) + Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "pod", "mlb-http", "--now")) + }) + + // Ensure the Service has a ready backend before hitting the VIP. + waitEndpoints := func() error { + rr, err := Run(t, exec.CommandContext( + ctx, "kubectl", "--context", profile, + "-n", "default", "get", "endpoints", "mlb-http-lb", + "-o", "jsonpath={.subsets[0].addresses[0].ip}")) + if err != nil { + return err + } + if strings.TrimSpace(rr.Stdout.String()) == "" { + return fmt.Errorf("endpoints not populated yet") + } + return nil + } + if err := retry.Expo(waitEndpoints, 1*time.Second, Minutes(2)); err != nil { + t.Fatalf("endpoints for mlb-http-lb never became ready: %v", err) + } + + // Wait for an external IP from MetalLB. + var lbIP string + waitExternalIP := func() error { + rr, err := Run(t, exec.CommandContext( + ctx, "kubectl", "--context", profile, + "-n", "default", "get", "svc", "mlb-http-lb", + "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}")) + if err != nil { + return err + } + ip := strings.TrimSpace(rr.Stdout.String()) + if ip == "" || strings.EqualFold(ip, "") { + // Fallback: some environments surface 'hostname' instead of 'ip' + rr2, err2 := Run(t, exec.CommandContext( + ctx, "kubectl", "--context", profile, + "-n", "default", "get", "svc", "mlb-http-lb", + "-o", "jsonpath={.status.loadBalancer.ingress[0].hostname}")) + if err2 != nil { + return err2 + } + host := strings.TrimSpace(rr2.Stdout.String()) + if host == "" || strings.EqualFold(host, "") { + return fmt.Errorf("external address not allocated yet") + } + lbIP = host + return nil + + } + lbIP = ip + return nil + } + + if err := retry.Expo(waitExternalIP, 2*time.Second, Minutes(8)); err != nil { + + t.Fatalf("failed waiting for LoadBalancer external IP: %v", err) + } + + t.Logf("mlb-http-lb external IP: %s", lbIP) + // Small grace for ARP announcement to propagate on some networks. + time.Sleep(2 * time.Second) + + // Verify it serves HTTP 200 from the host. + + endpoint := fmt.Sprintf("http://%s", lbIP) + checkLB := func() error { + resp, err := retryablehttp.Get(endpoint) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s = status %d, want 200", endpoint, resp.StatusCode) + } + return nil + } + if err := retry.Expo(checkLB, 500*time.Millisecond, Minutes(3)); err != nil { + t.Errorf("failed reaching LoadBalancer %s: %v", endpoint, err) + } +} + // validateIngressAddon tests the ingress addon by deploying a default nginx pod func validateIngressAddon(ctx context.Context, t *testing.T, profile string) { if NoneDriver() {