Prevent cluster autoscaler from reallocating job pods (#1919)

Author: vishalbollu

pkg/lib/k8s/job.go

@@ -58,8 +58,9 @@ func Job(spec *JobSpec) *kbatch.Job {
 			Parallelism:  &spec.Parallelism,
 			Template: kcore.PodTemplateSpec{
 				ObjectMeta: kmeta.ObjectMeta{
-					Name:   spec.PodSpec.Name,
-					Labels: spec.PodSpec.Labels,
+					Name:        spec.PodSpec.Name,
+					Labels:      spec.PodSpec.Labels,
+					Annotations: spec.PodSpec.Annotations,
 				},
 				Spec: spec.PodSpec.K8sPodSpec,
 			},

pkg/operator/resources/job/batchapi/k8s_specs.go

@@ -81,6 +81,7 @@ func pythonPredictorJobSpec(api *spec.API, job *spec.BatchJob) (*kbatch.Job, err
 			},
 			Annotations: map[string]string{
 				"traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0",
+				"cluster-autoscaler.kubernetes.io/safe-to-evict":   "false",
 			},
 			K8sPodSpec: kcore.PodSpec{
 				RestartPolicy: "Never",
@@ -130,6 +131,7 @@ func tensorFlowPredictorJobSpec(api *spec.API, job *spec.BatchJob) (*kbatch.Job,
 			},
 			Annotations: map[string]string{
 				"traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0",
+				"cluster-autoscaler.kubernetes.io/safe-to-evict":   "false",
 			},
 			K8sPodSpec: kcore.PodSpec{
 				RestartPolicy: "Never",
@@ -180,6 +182,7 @@ func onnxPredictorJobSpec(api *spec.API, job *spec.BatchJob) (*kbatch.Job, error
 			},
 			Annotations: map[string]string{
 				"traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0",
+				"cluster-autoscaler.kubernetes.io/safe-to-evict":   "false",
 			},
 			K8sPodSpec: kcore.PodSpec{
 				RestartPolicy: "Never",

pkg/operator/resources/job/taskapi/k8s_specs.go

@@ -109,6 +109,7 @@ func k8sJobSpec(api *spec.API, job *spec.TaskJob) (*kbatch.Job, error) {
 			},
 			Annotations: map[string]string{
 				"traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0",
+				"cluster-autoscaler.kubernetes.io/safe-to-evict":   "false",
 			},
 			K8sPodSpec: kcore.PodSpec{
 				RestartPolicy: "Never",