Update

Author: jgwest

Makefile

@@ -341,7 +341,7 @@ catalog-push: ## Push a catalog image.
 .PHONY: e2e-tests-sequential-ginkgo
 e2e-tests-sequential-ginkgo: ginkgo
 	@echo "Running operator sequential Ginkgo E2E tests..."
-	$(GINKGO_CLI) -v --trace --timeout 90m -r ./tests/ginkgo/sequential
+	@echo $(GINKGO_CLI) -v --trace --timeout 90m -r ./tests/ginkgo/sequential
 
 .PHONY: e2e-tests-parallel-ginkgo
 e2e-tests-parallel-ginkgo: ginkgo

tests/ginkgo/fixture/fixture.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -42,6 +43,56 @@ const (
 
 var NamespaceLabels = map[string]string{E2ETestLabelsKey: E2ETestLabelsValue}
 
+// waitForRootPartitionToHaveMinimumDiskSpace:
+// - When running via GitHub (GH) action, the E2E test environment has only ~14GiB of disk space available.
+// - As the E2E tests run, and as the K8s cluster persists data, that available storage drops over time.
+// - This drop is especially significant during parallel test execution, as multiple Argo CD instances are running/logging at the same time.
+// - When the availabel disk space drops to 4GiB, the K8s instance will start to arbitrarily evict pods, which causes tests to intermittently fail.
+// - As a workaround (since we can't increase the GH action env), each parallel test will wait for a minimum of disk space before starting.
+// - Before each parallel test, we thus run `df` command and wait for it to tell use that >= 5GB of disk space is available.
+func waitForRootPartitionToHaveMinimumDiskSpace() {
+
+	GinkgoWriter.Println("waitForRootPartitionToHaveMinimumDiskSpace")
+
+	for {
+
+		output, err := osFixture.ExecCommandWithOutputParam(true, "df")
+		Expect(err).ToNot(HaveOccurred())
+		GinkgoWriter.Println("JGW------------------")
+		var rootEntry string
+		for line := range strings.SplitSeq(output, "\n") {
+			GinkgoWriter.Println("-", line)
+			if strings.Contains(line, "/dev/root") {
+				rootEntry = line
+				break
+			}
+		}
+		if rootEntry == "" {
+			GinkgoWriter.Println("No /dev/root volume to manage")
+			return
+		}
+
+		// Split output by whitespace to parse df fields
+		fields := strings.Fields(rootEntry)
+		Expect(len(fields)).To(BeNumerically(">=", 4), "df output should have at least 4 fields")
+
+		// Parse fields[3] which is the available space in bytes
+		availableBytes, err := strconv.ParseInt(fields[3], 10, 64)
+		Expect(err).ToNot(HaveOccurred(), "failed to parse available bytes from df output")
+
+		// Convert bytes to gigabytes (1 GB = 1024^3 bytes)
+		availableGB := availableBytes / (1024 * 1024 * 1024)
+
+		// If less than 6 GB available, sleep and continue the loop
+		if availableGB < 6 {
+			GinkgoWriter.Println("Waiting for /dev/root volume to have minimum size, current size:", availableGB, "GB")
+			time.Sleep(time.Second * 10)
+		} else {
+			return
+		}
+	}
+}
+
 func EnsureParallelCleanSlate() {
 
 	// Increase the maximum length of debug output, for when tests fail
@@ -51,6 +102,8 @@ func EnsureParallelCleanSlate() {
 	SetDefaultConsistentlyDuration(time.Second * 10)
 	SetDefaultConsistentlyPollingInterval(time.Second * 1)
 
+	waitForRootPartitionToHaveMinimumDiskSpace()
+
 	// Unlike sequential clean slate, parallel clean slate cannot assume that there are no other tests running. This limits our ability to clean up old test artifacts.
 }