[no-relnote] enhance E2E

Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>

Author: ArangoGutierrez

tests/e2e/README.md

@@ -29,6 +29,14 @@ This repository contains a self‑contained Ginkgo v2 / Gomega end‑to‑end (
 On test failure the suite gathers logs and **ensures full cleanup** (namespace deletion, finalizer removal).
 The suite targets CI pipelines and developers validating chart or driver changes before promotion.
 
+### Key Features
+- **Ginkgo v2 idioms**: Uses `SpecContext`, `DeferCleanup`, `Label`, and `ReportAfterSuite`
+- **Parallel execution**: Label-based test sharding for faster CI runs
+- **Deterministic cleanup**: LIFO order cleanup with proper context handling
+- **No blind sleeps**: All waits use `Eventually` with explicit timeouts and context
+- **CI-ready reporting**: Built-in JSON and JUnit report generation
+- **GPU Feature Discovery**: Tests GFD label creation and NodeFeature API integration
+
 ---
 
 ## 2  Prerequisites
@@ -55,6 +63,8 @@ The suite targets CI pipelines and developers validating chart or driver changes
 | `LOG_ARTIFACTS_DIR` | ✖ | `./artifacts` | Directory for Helm & test logs. |
 | `COLLECT_LOGS_FROM` | ✖ | (unset) | Comma‑separated node list or `all` for log collection. |
 | `NVIDIA_DRIVER_ENABLED` | ✖ | `false` | Skip GPU job when driver is unavailable. |
+| `GINKGO_PARALLEL_PROCS` | ✖ | `4` | Number of parallel Ginkgo processes for parallel test execution. |
+| `GINKGO_FLAKE_ATTEMPTS` | ✖ | `2` | Number of retry attempts for flaky tests (should be reduced to 1 when stable). |
 
 > *Unset variables fall back to defaults via `getIntEnvVar` / `getBoolEnvVar`.*
 

tests/e2e/cleanup_test.go

@@ -48,19 +48,17 @@ func cleanupNamespaceResources(namespace string) {
 // waitForDeletion polls the provided checkFunc until a NotFound error is returned,
 // confirming that the resource is deleted.
 func waitForDeletion(resourceName string, checkFunc func() error) error {
-	timeout := 2 * time.Minute
-	interval := 5 * time.Second
-	start := time.Now()
-	for {
+	EventuallyWithOffset(1, func(g Gomega) error {
 		err := checkFunc()
 		if err != nil && errors.IsNotFound(err) {
 			return nil
 		}
-		if time.Since(start) > timeout {
-			return fmt.Errorf("timed out waiting for deletion of %s", resourceName)
+		if err != nil {
+			return err
 		}
-		time.Sleep(interval)
-	}
+		return fmt.Errorf("%s still exists", resourceName)
+	}).WithPolling(5 * time.Second).WithTimeout(2 * time.Minute).WithContext(ctx).Should(Succeed())
+	return nil
 }
 
 // cleanupTestPods deletes all test Pods in the namespace that have the label "app.nvidia.com=k8s-dra-driver-gpu-test-app".
@@ -224,14 +222,10 @@ func cleanupNode(cs clientset.Interface) {
 	Expect(err).NotTo(HaveOccurred())
 
 	for _, n := range nodeList.Items {
-		var err error
-		for retry := 0; retry < 5; retry++ {
-			if err = cleanup(n.Name); err == nil {
-				break
-			}
-			time.Sleep(100 * time.Millisecond)
-		}
-		Expect(err).NotTo(HaveOccurred())
+		nodeName := n.Name
+		Eventually(func(g Gomega) error {
+			return cleanup(nodeName)
+		}).WithPolling(100 * time.Millisecond).WithTimeout(500 * time.Millisecond).Should(Succeed())
 	}
 }
 

tests/e2e/device-plugin_test.go

@@ -18,7 +18,6 @@
 package e2e
 
 import (
-	"context"
 	"fmt"
 	"strings"
 	"time"
@@ -30,6 +29,7 @@ import (
 	helmValues "github.com/mittwald/go-helm-client/values"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/NVIDIA/k8s-device-plugin/tests/e2e/internal"
 	"github.com/NVIDIA/k8s-test-infra/pkg/diagnostics"
 )
 
@@ -38,7 +38,7 @@ const (
 )
 
 // Actual test suite
-var _ = Describe("GPU Device Plugin", Ordered, func() {
+var _ = Describe("GPU Device Plugin", Ordered, Label("gpu", "e2e", "device-plugin"), func() {
 	// Init global suite vars vars
 	var (
 		helmReleaseName string
@@ -75,7 +75,7 @@ var _ = Describe("GPU Device Plugin", Ordered, func() {
 		collectLogsFrom = strings.Split(CollectLogsFrom, ",")
 	}
 
-	BeforeAll(func(ctx context.Context) {
+	BeforeAll(func(ctx SpecContext) {
 		// Create clients for apiextensions and our CRD api
 		helmReleaseName = "nvdp-e2e-test-" + randomSuffix()
 
@@ -92,9 +92,24 @@ var _ = Describe("GPU Device Plugin", Ordered, func() {
 		By("Installing k8s-device-plugin Helm chart")
 		_, err := helmClient.InstallChart(ctx, &chartSpec, nil)
 		Expect(err).NotTo(HaveOccurred())
+
+		// Wait for all DaemonSets to be ready
+		// Note: DaemonSet names are dynamically generated with the Helm release prefix,
+		// so we wait for all DaemonSets in the namespace rather than specific names
+		By("Waiting for all DaemonSets to be ready")
+		err = internal.WaitForAllDaemonSetsReady(ctx, clientSet, testNamespace.Name)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterAll(func(ctx SpecContext) {
+		By("Uninstalling k8s-device-plugin Helm chart")
+		err := helmClient.UninstallReleaseByName(helmReleaseName)
+		if err != nil {
+			GinkgoWriter.Printf("Failed to uninstall helm release %s: %v\n", helmReleaseName, err)
+		}
 	})
 
-	AfterEach(func(ctx context.Context) {
+	AfterEach(func(ctx SpecContext) {
 		// Run diagnostic collector if test failed
 		if CurrentSpecReport().Failed() {
 			var err error
@@ -111,18 +126,8 @@ var _ = Describe("GPU Device Plugin", Ordered, func() {
 		}
 	})
 
-	AfterAll(func(ctx context.Context) {
-		By("Deleting the job")
-		job, err := clientSet.BatchV1().Jobs(testNamespace.Name).List(ctx, metav1.ListOptions{})
-		Expect(err).NotTo(HaveOccurred())
-		Expect(len(job.Items)).ToNot(BeZero())
-
-		err = clientSet.BatchV1().Jobs(testNamespace.Name).Delete(ctx, job.Items[0].Name, metav1.DeleteOptions{})
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	When("When deploying k8s-device-plugin", Ordered, func() {
-		It("it should create nvidia.com/gpu resource", func(ctx context.Context) {
+	When("When deploying k8s-device-plugin", Ordered, Label("serial"), func() {
+		It("it should create nvidia.com/gpu resource", Label("gpu-resource"), func(ctx SpecContext) {
 			nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(nodeList.Items)).ToNot(BeZero())
@@ -141,25 +146,35 @@ var _ = Describe("GPU Device Plugin", Ordered, func() {
 				}}
 			eventuallyNonControlPlaneNodes(ctx, clientSet).Should(MatchCapacity(capacityChecker, nodes), "Node capacity does not match")
 		})
-		It("it should run GPU jobs", func(ctx context.Context) {
+		It("it should run GPU jobs", Label("gpu-job"), func(ctx SpecContext) {
 			By("Creating a GPU job")
-			job, err := CreateOrUpdateJobsFromFile(ctx, clientSet, "job-1.yaml", testNamespace.Name)
+			jobNames, err := CreateOrUpdateJobsFromFile(ctx, clientSet, "job-1.yaml", testNamespace.Name)
 			Expect(err).NotTo(HaveOccurred())
+			Expect(jobNames).NotTo(BeEmpty())
+
+			// Defer cleanup for the job
+			DeferCleanup(func(ctx SpecContext) {
+				By("Deleting the GPU job")
+				err := clientSet.BatchV1().Jobs(testNamespace.Name).Delete(ctx, jobNames[0], metav1.DeleteOptions{})
+				if err != nil {
+					GinkgoWriter.Printf("Failed to delete job %s: %v\n", jobNames[0], err)
+				}
+			})
 
 			By("Waiting for job to complete")
-			Eventually(func() error {
-				job, err := clientSet.BatchV1().Jobs(testNamespace.Name).Get(ctx, job[0], metav1.GetOptions{})
+			Eventually(func(g Gomega) error {
+				job, err := clientSet.BatchV1().Jobs(testNamespace.Name).Get(ctx, jobNames[0], metav1.GetOptions{})
 				if err != nil {
 					return err
 				}
-				if job.Status.Succeeded != 1 {
-					return fmt.Errorf("job %s/%s failed", job.Namespace, job.Name)
+				if job.Status.Failed > 0 {
+					return fmt.Errorf("job %s/%s has failed pods: %d", job.Namespace, job.Name, job.Status.Failed)
 				}
-				if job.Status.Succeeded == 1 {
-					return nil
+				if job.Status.Succeeded != 1 {
+					return fmt.Errorf("job %s/%s not completed yet: %d succeeded", job.Namespace, job.Name, job.Status.Succeeded)
 				}
-				return fmt.Errorf("job %s/%s not completed yet", job.Namespace, job.Name)
-			}, devicePluginEventuallyTimeout, 5*time.Second).Should(BeNil())
+				return nil
+			}).WithContext(ctx).WithPolling(5 * time.Second).WithTimeout(devicePluginEventuallyTimeout).Should(Succeed())
 		})
 	})
 })

tests/e2e/e2e_test.go

@@ -103,13 +103,17 @@ func TestMain(t *testing.T) {
 	ctx = context.Background()
 	getTestEnv()
 
+	// Log random seed for reproducibility
+	GinkgoWriter.Printf("Random seed: %d\n", GinkgoRandomSeed())
+
 	RunSpecs(t,
 		suiteName,
+		Label("e2e"),
 	)
 }
 
 // BeforeSuite runs before the test suite
-var _ = BeforeSuite(func() {
+var _ = BeforeSuite(func(ctx SpecContext) {
 	var err error
 
 	cwd, err = os.Getwd()
@@ -130,16 +134,31 @@ var _ = BeforeSuite(func() {
 	getHelmClient()
 })
 
-var _ = AfterSuite(func() {
+var _ = AfterSuite(func(ctx SpecContext) {
 	By("Cleaning up namespace resources")
-	// Remove finalizers and force delete resourceclaims, resourceclaimtemplates, daemonsets, and pods.
 	cleanupNamespaceResources(testNamespace.Name)
 
 	By("Deleting the test namespace")
-	// Delete the test namespace to remove any remaining objects.
 	deleteTestNamespace()
 })
 
+// Add ReportAfterSuite for logging test summary and random seed
+var _ = ReportAfterSuite("", func(report Report) {
+	// Log test summary
+	failedCount := 0
+	for _, specReport := range report.SpecReports {
+		if specReport.Failed() {
+			failedCount++
+		}
+	}
+
+	GinkgoWriter.Printf("\nTest Summary:\n")
+	GinkgoWriter.Printf("  Total Specs: %d\n", len(report.SpecReports))
+	GinkgoWriter.Printf("  Random Seed: %d\n", report.SuiteConfig.RandomSeed)
+	GinkgoWriter.Printf("  Failed: %d\n", failedCount)
+	GinkgoWriter.Printf("  Duration: %.2fs\n", report.RunTime.Seconds())
+})
+
 // getK8sClients creates the k8s clients
 func getK8sClients() {
 	var err error
@@ -297,7 +316,7 @@ type k8sLabels map[string]string
 //
 //nolint:unused
 func eventuallyNonControlPlaneNodes(ctx context.Context, cli clientset.Interface) AsyncAssertion {
-	return Eventually(func(g Gomega, ctx context.Context) ([]corev1.Node, error) {
+	return Eventually(func(g Gomega) ([]corev1.Node, error) {
 		return getNonControlPlaneNodes(ctx, cli)
 	}).WithPolling(1 * time.Second).WithTimeout(1 * time.Minute).WithContext(ctx)
 }

tests/e2e/gpu-feature-discovery_test.go

@@ -31,11 +31,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/rand"
 
+	"github.com/NVIDIA/k8s-device-plugin/tests/e2e/internal"
 	"github.com/NVIDIA/k8s-test-infra/pkg/diagnostics"
 )
 
 // Actual test suite
-var _ = Describe("GPU Feature Discovery", Ordered, func() {
+var _ = Describe("GPU Feature Discovery", Ordered, Label("gfd", "gpu", "e2e"), func() {
 	expectedLabelPatterns := k8sLabels{
 		"nvidia.com/gfd.timestamp":        "[0-9]{10}",
 		"nvidia.com/cuda.driver.major":    "[0-9]+",
@@ -97,7 +98,7 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 		collectLogsFrom = strings.Split(CollectLogsFrom, ",")
 	}
 
-	BeforeAll(func(ctx context.Context) {
+	BeforeAll(func(ctx SpecContext) {
 		helmReleaseName = "gfd-e2e-test" + rand.String(5)
 
 		// reset Helm Client
@@ -114,10 +115,25 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 		By("Installing GFD Helm chart")
 		_, err := helmClient.InstallChart(ctx, &chartSpec, nil)
 		Expect(err).NotTo(HaveOccurred())
+
+		// Wait for all DaemonSets to be ready
+		// Note: DaemonSet names are dynamically generated with the Helm release prefix,
+		// so we wait for all DaemonSets in the namespace rather than specific names
+		By("Waiting for all DaemonSets to be ready")
+		err = internal.WaitForAllDaemonSetsReady(ctx, clientSet, testNamespace.Name)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterAll(func(ctx SpecContext) {
+		By("Uninstalling GFD Helm chart")
+		err := helmClient.UninstallReleaseByName(helmReleaseName)
+		if err != nil {
+			GinkgoWriter.Printf("Failed to uninstall helm release %s: %v\n", helmReleaseName, err)
+		}
 	})
 
 	// Cleanup before next test run
-	AfterEach(func(ctx context.Context) {
+	AfterEach(func(ctx SpecContext) {
 		// Run diagnostic collector if test failed
 		if CurrentSpecReport().Failed() {
 			var err error
@@ -135,9 +151,17 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 		}
 	})
 
-	When("When deploying GFD", Ordered, func() {
+	When("When deploying GFD", Ordered, Label("serial"), func() {
 		Context("NV Driver is not installed", func() {
-			It("it should create nvidia.com timestamp label", func(ctx context.Context) {
+			BeforeEach(func() {
+				// Skip this context when driver is enabled since "NV Driver is installed"
+				// context provides more comprehensive testing
+				if NVIDIA_DRIVER_ENABLED {
+					Skip("Skipping driver-not-installed tests when NVIDIA_DRIVER_ENABLED is true")
+				}
+			})
+
+			It("it should create nvidia.com timestamp label", Label("timestamp"), func(ctx SpecContext) {
 				nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(nodeList.Items)).ToNot(BeZero())
@@ -157,7 +181,7 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 				eventuallyNonControlPlaneNodes(ctx, clientSet).Should(MatchLabels(labelChecker, nodes))
 			})
 			Context("and the NodeFeature API is enabled", func() {
-				It("gfd should create node feature object", func(ctx context.Context) {
+				It("gfd should create node feature object", Label("nodefeature"), func(ctx SpecContext) {
 					By("Updating GFD Helm chart values")
 					newValues := values
 					newValues.Values = append(newValues.Values, "nfd.enableNodeFeatureApi=true")
@@ -172,9 +196,9 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 
 					targetNodeName := nodes[0].Name
 					Expect(targetNodeName).ToNot(BeEmpty())
-					Eventually(func() bool {
+					Eventually(func(g Gomega) bool {
 						return checkNodeFeatureObject(ctx, targetNodeName)
-					}, 2*time.Minute, 5*time.Second).Should(BeTrue())
+					}).WithContext(ctx).WithPolling(5 * time.Second).WithTimeout(2 * time.Minute).Should(BeTrue())
 
 					By("Checking that node labels are created from NodeFeature object")
 					labelChecker := map[string]k8sLabels{
@@ -187,7 +211,7 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 		})
 
 		When("NV Driver is installed", func() {
-			It("it should create nvidia.com labels", func(ctx context.Context) {
+			It("it should create nvidia.com labels", Label("driver", "labels"), func(ctx SpecContext) {
 				if !NVIDIA_DRIVER_ENABLED {
 					Skip("NVIDIA_DRIVER_ENABLED is not set")
 				}
@@ -210,7 +234,7 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 				eventuallyNonControlPlaneNodes(ctx, clientSet).Should(MatchLabels(labelChecker, nodes))
 			})
 			Context("and the NodeFeature API is enabled", func() {
-				It("gfd should create node feature object", func(ctx context.Context) {
+				It("gfd should create node feature object", Label("driver", "nodefeature"), func(ctx SpecContext) {
 					if !NVIDIA_DRIVER_ENABLED {
 						Skip("NVIDIA_DRIVER_ENABLED is not set")
 					}
@@ -228,9 +252,9 @@ var _ = Describe("GPU Feature Discovery", Ordered, func() {
 
 					targetNodeName := nodes[0].Name
 					Expect(targetNodeName).ToNot(BeEmpty())
-					Eventually(func() bool {
+					Eventually(func(g Gomega) bool {
 						return checkNodeFeatureObject(ctx, targetNodeName)
-					}, 2*time.Minute, 5*time.Second).Should(BeTrue())
+					}).WithContext(ctx).WithPolling(5 * time.Second).WithTimeout(2 * time.Minute).Should(BeTrue())
 
 					By("Checking that node labels are created from NodeFeature CR object")
 					checkForLabels := map[string]k8sLabels{