OOM (Out of memory) status (#40)

Author: 1vn

pkg/api/resource/saved_status.go

@@ -47,6 +47,7 @@ const (
 	ExitCodeDataSucceeded DataExitCode = "succeeded"
 	ExitCodeDataFailed    DataExitCode = "failed"
 	ExitCodeDataKilled    DataExitCode = "killed"
+	ExitCodeDataOOM       DataExitCode = "oom"
 )
 
 func DataSavedStatusPtrsEqual(savedStatus *DataSavedStatus, savedStatus2 *DataSavedStatus) bool {

pkg/api/resource/status.go

@@ -109,6 +109,8 @@ const (
 	StatusAPIGroupParentFailed
 	StatusAPIGroupParentKilled
 	StatusAPIGroupUpdateSkipped
+
+	StatusDataKilledOOM
 )
 
 var statusCodes = []string{
@@ -136,9 +138,11 @@ var statusCodes = []string{
 	"status_api_group_parent_failed",
 	"status_api_group_parent_killed",
 	"status_api_group_update_skipped",
+
+	"status_data_oom",
 }
 
-var _ = [1]int{}[int(StatusAPIGroupUpdateSkipped)-(len(statusCodes)-1)] // Ensure list length matches
+var _ = [1]int{}[int(StatusDataKilledOOM)-(len(statusCodes)-1)] // Ensure list length matches
 
 var statusCodeMessages = []string{
 	"unknown", // StatusUnknown
@@ -165,9 +169,11 @@ var statusCodeMessages = []string{
 	"upstream error",       // StatusAPIGroupParentFailed
 	"upstream termination", // StatusAPIGroupParentKilled
 	"update skipped",       // StatusAPIGroupUpdateSkipped
+
+	"terminated (out of mem)", // StatusDataOOM
 }
 
-var _ = [1]int{}[int(StatusAPIGroupUpdateSkipped)-(len(statusCodeMessages)-1)] // Ensure list length matches
+var _ = [1]int{}[int(StatusDataKilledOOM)-(len(statusCodeMessages)-1)] // Ensure list length matches
 
 // StatusDataRunning aliases
 const (
@@ -203,9 +209,11 @@ var statusSortBuckets = []int{
 	2, // StatusAPIGroupParentFailed
 	2, // StatusAPIGroupParentKilled
 	2, // StatusAPIGroupUpdateSkipped
+
+	1, // StatusDataKilledOOM
 }
 
-var _ = [1]int{}[int(StatusAPIGroupUpdateSkipped)-(len(statusSortBuckets)-1)] // Ensure list length matches
+var _ = [1]int{}[int(StatusDataKilledOOM)-(len(statusSortBuckets)-1)] // Ensure list length matches
 
 func (code StatusCode) String() string {
 	if int(code) < 0 || int(code) >= len(statusCodes) {

pkg/operator/k8s/pod.go

@@ -33,13 +33,14 @@ var podTypeMeta = metav1.TypeMeta{
 }
 
 const (
+	PodStatusUnknown     = "Unknown"
 	PodStatusPending     = "Pending"
 	PodStatusRunning     = "Running"
 	PodStatusTerminating = "Terminating"
 	PodStatusSucceeded   = "Succeeded"
 	PodStatusFailed      = "Failed"
 	PodStatusKilled      = "Killed"
-	PodStatusUnknown     = "Unknown"
+	PodStatusKilledOOM   = "Out of Memory"
 )
 
 var killStatuses = map[int32]bool{
@@ -108,11 +109,17 @@ func GetPodStatus(pod *corev1.Pod) string {
 		for _, containerStatus := range pod.Status.ContainerStatuses {
 			if containerStatus.LastTerminationState.Terminated != nil {
 				exitCode := containerStatus.LastTerminationState.Terminated.ExitCode
+				if exitCode == 137 {
+					return PodStatusKilledOOM
+				}
 				if killStatuses[exitCode] {
 					return PodStatusKilled
 				}
 			} else if containerStatus.State.Terminated != nil {
 				exitCode := containerStatus.State.Terminated.ExitCode
+				if exitCode == 137 {
+					return PodStatusKilledOOM
+				}
 				if killStatuses[exitCode] {
 					return PodStatusKilled
 				}

pkg/operator/workloads/api_status.go

@@ -194,7 +194,7 @@ func updateAPIStatusCodeByParents(apiStatus *resource.APIStatus, dataStatuses ma
 	parentSkipped := false
 	for dependency := range allDependencies {
 		switch dataStatuses[dependency].Code {
-		case resource.StatusDataKilled:
+		case resource.StatusDataKilled, resource.StatusDataKilledOOM:
 			apiStatus.Code = resource.StatusParentKilled
 			return
 		case resource.StatusDataFailed:

pkg/operator/workloads/data_status.go

@@ -77,6 +77,8 @@ func dataStatusCode(dataSavedStatus *resource.DataSavedStatus) resource.StatusCo
 		return resource.StatusDataFailed
 	case resource.ExitCodeDataKilled:
 		return resource.StatusDataKilled
+	case resource.ExitCodeDataOOM:
+		return resource.StatusDataKilledOOM
 	}
 
 	return resource.StatusUnknown
@@ -91,7 +93,7 @@ func updateDataStatusCodeByParents(dataStatus *resource.DataStatus, dataStatuses
 	parentSkipped := false
 	for dependency := range allDependencies {
 		switch dataStatuses[dependency].Code {
-		case resource.StatusDataKilled:
+		case resource.StatusDataKilled, resource.StatusDataKilledOOM:
 			dataStatus.Code = resource.StatusParentKilled
 			return
 		case resource.StatusDataFailed:

pkg/operator/workloads/workload_spec.go

@@ -150,9 +150,13 @@ func UpdateDataWorkflowErrors(failedPods []corev1.Pod) error {
 					savedStatus.Start = nowTime
 				}
 
-				savedStatus.ExitCode = resource.ExitCodeDataFailed
-				if k8s.GetPodStatus(&pod) == k8s.PodStatusKilled {
+				switch k8s.GetPodStatus(&pod) {
+				case k8s.PodStatusKilled:
 					savedStatus.ExitCode = resource.ExitCodeDataKilled
+				case k8s.PodStatusKilledOOM:
+					savedStatus.ExitCode = resource.ExitCodeDataOOM
+				default:
+					savedStatus.ExitCode = resource.ExitCodeDataFailed
 				}
 
 				savedStatusesToUpload = append(savedStatusesToUpload, savedStatus)