test: Upload job logs

Author: astefanutti

.github/workflows/e2e_tests.yaml

@@ -169,6 +169,9 @@ jobs:
           export CODEFLARE_TEST_TIMEOUT_MEDIUM=3m
           export CODEFLARE_TEST_TIMEOUT_LONG=8m
 
+          export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
+          echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
+
           make test-e2e
 
       - name: Print CodeFlare operator logs
@@ -188,3 +191,12 @@ jobs:
         run: |
           echo "Printing KubeRay operator logs"
           kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: always() && steps.deploy.outcome == 'success'
+        with:
+          name: logs
+          retention-days: 10
+          path: |
+            ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log

test/e2e/mnist_pytorch_mcad_job_test.go

@@ -151,8 +151,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
 			WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
 		))
 
-	// Print the job logs
-	PrintJobLogs(test, job.Namespace, job.Name)
+	// Retrieve the job logs
+	WriteJobLogs(test, job.Namespace, job.Name)
 
 	// Assert the job has completed successfully
 	test.Expect(GetJob(test, job.Namespace, job.Name)).

test/e2e/mnist_raycluster_sdk_test.go

@@ -195,8 +195,8 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 			WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)),
 		))
 
-	// Print the job logs
-	PrintJobLogs(test, job.Namespace, job.Name)
+	// Retrieve the job logs
+	WriteJobLogs(test, job.Namespace, job.Name)
 
 	// Assert the job has completed successfully
 	test.Expect(GetJob(test, job.Namespace, job.Name)).

test/e2e/mnist_rayjob_mcad_raycluster_test.go

@@ -254,8 +254,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) {
 	test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong).
 		Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal)))
 
-	test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name)
-	test.T().Log(GetRayJobLogs(test, rayJob.Namespace, rayJob.Name))
+	test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name)
+	WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name))
 
 	// Assert the Ray job has completed successfully
 	test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)).

test/e2e/support.go

@@ -28,6 +28,7 @@ import (
 var files embed.FS
 
 func ReadFile(t support.Test, fileName string) []byte {
+	t.T().Helper()
 	file, err := files.ReadFile(fileName)
 	t.Expect(err).NotTo(gomega.HaveOccurred())
 	return file

test/support/batch.go

@@ -38,7 +38,7 @@ func GetJob(t Test, namespace, name string) *batchv1.Job {
 	return Job(t, namespace, name)(t)
 }
 
-func PrintJobLogs(t Test, namespace, name string) {
+func WriteJobLogs(t Test, namespace, name string) {
 	t.T().Helper()
 
 	job := GetJob(t, namespace, name)
@@ -51,8 +51,8 @@ func PrintJobLogs(t Test, namespace, name string) {
 		t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
 	} else {
 		for i, pod := range pods {
-			t.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name)
-			t.T().Log(GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
+			t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name)
+			WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
 		}
 	}
 }

test/support/codeflare.go

@@ -20,13 +20,18 @@ import (
 	"os"
 )
 
-// The environment variables hereafter can be used to change the components
-// used for testing.
 const (
+	// The environment variables hereafter can be used to change the components
+	// used for testing.
+
 	CodeFlareTestSdkVersion   = "CODEFLARE_TEST_SDK_VERSION"
 	CodeFlareTestRayVersion   = "CODEFLARE_TEST_RAY_VERSION"
 	CodeFlareTestRayImage     = "CODEFLARE_TEST_RAY_IMAGE"
 	CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE"
+
+	// The testing output directory, to write output files into.
+
+	CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR"
 )
 
 func GetCodeFlareSDKVersion() string {

test/support/core.go

@@ -43,7 +43,7 @@ func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod
 	return pods.Items
 }
 
-func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string {
+func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte {
 	t.T().Helper()
 	stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx())
 	t.Expect(err).NotTo(gomega.HaveOccurred())
@@ -55,5 +55,5 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string {
 	bytes, err := io.ReadAll(stream)
 	t.Expect(err).NotTo(gomega.HaveOccurred())
 
-	return string(bytes)
+	return bytes
 }

test/support/ray.go

@@ -45,7 +45,7 @@ func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus {
 	return job.Status.JobStatus
 }
 
-func GetRayJobLogs(t Test, namespace, name string) string {
+func GetRayJobLogs(t Test, namespace, name string) []byte {
 	t.T().Helper()
 
 	job := GetRayJob(t, namespace, name)
@@ -61,5 +61,5 @@ func GetRayJobLogs(t Test, namespace, name string) string {
 	t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed())
 	t.Expect(body).To(gomega.HaveKey("logs"))
 
-	return body["logs"]
+	return []byte(body["logs"])
 }

test/support/test.go

@@ -18,6 +18,8 @@ package support
 
 import (
 	"context"
+	"os"
+	"path"
 	"sync"
 	"testing"
 
@@ -30,6 +32,7 @@ type Test interface {
 	T() *testing.T
 	Ctx() context.Context
 	Client() Client
+	OutputDir() string
 
 	gomega.Gomega
 
@@ -70,9 +73,13 @@ type T struct {
 	*gomega.WithT
 	t *testing.T
 	// nolint: containedctx
-	ctx    context.Context
-	client Client
-	once   sync.Once
+	ctx       context.Context
+	client    Client
+	outputDir string
+	once      struct {
+		client    sync.Once
+		outputDir sync.Once
+	}
 }
 
 func (t *T) T() *testing.T {
@@ -84,7 +91,8 @@ func (t *T) Ctx() context.Context {
 }
 
 func (t *T) Client() Client {
-	t.once.Do(func() {
+	t.T().Helper()
+	t.once.client.Do(func() {
 		c, err := newTestClient()
 		if err != nil {
 			t.T().Fatalf("Error creating client: %v", err)
@@ -94,6 +102,31 @@ func (t *T) Client() Client {
 	return t.client
 }
 
+func (t *T) OutputDir() string {
+	t.T().Helper()
+	t.once.outputDir.Do(func() {
+		if parent, ok := os.LookupEnv(CodeFlareTestOutputDir); ok {
+			if !path.IsAbs(parent) {
+				if cwd, err := os.Getwd(); err == nil {
+					// best effort to output the parent absolute path
+					parent = path.Join(cwd, parent)
+				}
+			}
+			t.T().Logf("Creating output directory in parent directory: %s", parent)
+			dir, err := os.MkdirTemp(parent, t.T().Name())
+			if err != nil {
+				t.T().Fatalf("Error creating output directory: %v", err)
+			}
+			t.outputDir = dir
+		} else {
+			t.T().Logf("Creating ephemeral output directory as %s env variable is unset", CodeFlareTestOutputDir)
+			t.outputDir = t.T().TempDir()
+		}
+		t.T().Logf("Output directory has been created at: %s", t.outputDir)
+	})
+	return t.outputDir
+}
+
 func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Namespace {
 	t.T().Helper()
 	namespace := createTestNamespace(t, options...)