test: Add defered troubleshooting logs

Author: astefanutti

test/e2e/mnist_pytorch_mcad_job_test.go

@@ -182,15 +182,17 @@ torchvision==0.12.0
 	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
 		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
 
+	defer troubleshooting(test, job)
+
 	test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name)
-	test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong).
+	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).
 		Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))
 
 	// Refresh the job to get the generated pod selector
-	job = GetJob(test, namespace, job.Name)
+	job = GetJob(test, job.Namespace, job.Name)
 
 	// Get the job Pod
-	pods := GetPods(test, namespace, metav1.ListOptions{
+	pods := GetPods(test, job.Namespace, metav1.ListOptions{
 		LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
 	)
 	test.Expect(pods).To(HaveLen(1))
@@ -199,3 +201,25 @@ torchvision==0.12.0
 	test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name)
 	test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{}))
 }
+
+func troubleshooting(test Test, job *batchv1.Job) {
+	if !test.T().Failed() {
+		return
+	}
+	job = GetJob(test, job.Namespace, job.Name)
+
+	test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job)
+
+	pods := GetPods(test, job.Namespace, metav1.ListOptions{
+		LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
+	)
+
+	if len(pods) == 0 {
+		test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
+	} else {
+		for i, pod := range pods {
+			test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name)
+			test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{}))
+		}
+	}
+}

test/support/batch.go

@@ -20,19 +20,18 @@ import (
 	"github.com/onsi/gomega"
 
 	batchv1 "k8s.io/api/batch/v1"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func Job(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *batchv1.Job {
+func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job {
 	return func(g gomega.Gomega) *batchv1.Job {
-		job, err := t.Client().Core().BatchV1().Jobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{})
+		job, err := t.Client().Core().BatchV1().Jobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{})
 		g.Expect(err).NotTo(gomega.HaveOccurred())
 		return job
 	}
 }
 
-func GetJob(t Test, namespace *corev1.Namespace, name string) *batchv1.Job {
+func GetJob(t Test, namespace, name string) *batchv1.Job {
 	t.T().Helper()
 	return Job(t, namespace, name)(t)
 }

test/support/core.go

@@ -36,9 +36,9 @@ func Raw(t Test, obj runtime.Object) runtime.RawExtension {
 	}
 }
 
-func GetPods(t Test, namespace *corev1.Namespace, options metav1.ListOptions) []corev1.Pod {
+func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod {
 	t.T().Helper()
-	pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), options)
+	pods, err := t.Client().Core().CoreV1().Pods(namespace).List(t.Ctx(), options)
 	t.Expect(err).NotTo(gomega.HaveOccurred())
 	return pods.Items
 }