Add additional volumes to pgbackrest pods.

Change the API to allow users to specify preexisting PVCs to attach to specified containers in the various pgbackrest related pods: backup jobs, repo hosts, and restore jobs.

The spec allos users to specify whether to add the volume to:
* all containers (by omitting the containers list)
* no containers (by specifying an empty containers list)
* a list of named containers

If any of the named containers isnt present, we continue to reconcile, but issue a warning event with the names of the missing containers.

Issues: [PGO-2564]

Author: dsessler7

config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -608,7 +608,8 @@ func (r *Reconciler) generateRepoHostIntent(ctx context.Context, postgresCluster
 		},
 	}
 
-	if repoHost := postgresCluster.Spec.Backups.PGBackRest.RepoHost; repoHost != nil {
+	repoHost := postgresCluster.Spec.Backups.PGBackRest.RepoHost
+	if repoHost != nil {
 		repo.Spec.Template.Spec.Affinity = repoHost.Affinity
 		repo.Spec.Template.Spec.Tolerations = repoHost.Tolerations
 		repo.Spec.Template.Spec.TopologySpreadConstraints = repoHost.TopologySpreadConstraints
@@ -718,6 +719,16 @@ func (r *Reconciler) generateRepoHostIntent(ctx context.Context, postgresCluster
 
 	AddTMPEmptyDir(&repo.Spec.Template)
 
+	// mount additional volumes to the repo host containers
+	if repoHost != nil && repoHost.Volumes != nil && len(repoHost.Volumes.Additional) > 0 {
+		missingContainers := addAdditionalVolumesToSpecifiedContainers(&repo.Spec.Template, repoHost.Volumes.Additional)
+
+		if len(missingContainers) > 0 {
+			r.Recorder.Eventf(postgresCluster, corev1.EventTypeWarning, "SpecifiedContainerNotFound",
+				"The following Repo Host Pod containers were specified for additional volumes but cannot be found: %s.", missingContainers)
+		}
+	}
+
 	// set ownership references
 	if err := r.setControllerReference(postgresCluster, repo); err != nil {
 		return nil, err
@@ -814,7 +825,8 @@ func (r *Reconciler) generateBackupJobSpecIntent(ctx context.Context, postgresCl
 		container.Command = append(container.Command, cmdOpts...)
 	}
 
-	if postgresCluster.Spec.Backups.PGBackRest.Jobs != nil {
+	jobs := postgresCluster.Spec.Backups.PGBackRest.Jobs
+	if jobs != nil {
 		container.Resources = postgresCluster.Spec.Backups.PGBackRest.Jobs.Resources
 	}
 
@@ -848,12 +860,9 @@ func (r *Reconciler) generateBackupJobSpecIntent(ctx context.Context, postgresCl
 		},
 	}
 
-	if jobs := postgresCluster.Spec.Backups.PGBackRest.Jobs; jobs != nil {
+	// set the job lifetime, priority class name, tolerations, and affinity, if they exist
+	if jobs != nil {
 		jobSpec.TTLSecondsAfterFinished = jobs.TTLSecondsAfterFinished
-	}
-
-	// set the priority class name, tolerations, and affinity, if they exist
-	if postgresCluster.Spec.Backups.PGBackRest.Jobs != nil {
 		jobSpec.Template.Spec.Tolerations = postgresCluster.Spec.Backups.PGBackRest.Jobs.Tolerations
 		jobSpec.Template.Spec.Affinity = postgresCluster.Spec.Backups.PGBackRest.Jobs.Affinity
 		jobSpec.Template.Spec.PriorityClassName =
@@ -897,6 +906,16 @@ func (r *Reconciler) generateBackupJobSpecIntent(ctx context.Context, postgresCl
 		}
 	}
 
+	// mount additional volumes to the job containers
+	if jobs != nil && jobs.Volumes != nil && len(jobs.Volumes.Additional) > 0 {
+		missingContainers := addAdditionalVolumesToSpecifiedContainers(&jobSpec.Template, jobs.Volumes.Additional)
+
+		if len(missingContainers) > 0 {
+			r.Recorder.Eventf(postgresCluster, corev1.EventTypeWarning, "SpecifiedContainerNotFound",
+				"The following Backup Job Pod containers were specified for additional volumes but cannot be found: %s.", missingContainers)
+		}
+	}
+
 	return jobSpec
 }
 
@@ -1388,6 +1407,15 @@ func (r *Reconciler) generateRestoreJobIntent(cluster *v1beta1.PostgresCluster,
 	// set the priority class name, if it exists
 	job.Spec.Template.Spec.PriorityClassName = initialize.FromPointer(dataSource.PriorityClassName)
 
+	if dataSource.Volumes != nil && len(dataSource.Volumes.Additional) > 0 {
+		missingContainers := addAdditionalVolumesToSpecifiedContainers(&job.Spec.Template, dataSource.Volumes.Additional)
+
+		if len(missingContainers) > 0 {
+			r.Recorder.Eventf(cluster, corev1.EventTypeWarning, "SpecifiedContainerNotFound",
+				"The following Restore Pod containers were specified for additional volumes but cannot be found: %s.", missingContainers)
+		}
+	}
+
 	job.SetGroupVersionKind(batchv1.SchemeGroupVersion.WithKind("Job"))
 	if err := errors.WithStack(r.setControllerReference(cluster, job)); err != nil {
 		return err
@@ -1821,6 +1849,7 @@ func (r *Reconciler) reconcileCloudBasedDataSource(ctx context.Context,
 		Affinity:          dataSource.Affinity,
 		Tolerations:       dataSource.Tolerations,
 		PriorityClassName: dataSource.PriorityClassName,
+		Volumes:           dataSource.Volumes,
 	}
 
 	// reconcile the pgBackRest restore Job to populate the cluster's data directory

internal/controller/postgrescluster/pgbackrest.go

@@ -3070,6 +3070,46 @@ volumes:
 		// No events created
 		assert.Equal(t, len(recorder.Events), 0)
 	})
+
+	t.Run("AdditionalVolumes", func(t *testing.T) {
+		recorder := events.NewRecorder(t, runtime.Scheme)
+		r.Recorder = recorder
+
+		cluster := cluster.DeepCopy()
+		cluster.Namespace = ns.Name
+		cluster.Spec.Backups.PGBackRest.Jobs = &v1beta1.BackupJobs{
+			Volumes: &v1beta1.PGBackRestVolumesSpec{
+				Additional: []v1beta1.AdditionalVolume{
+					{
+						ClaimName: "additional-pvc",
+						Name:      "stuff",
+					},
+				},
+			},
+		}
+
+		spec := r.generateBackupJobSpecIntent(ctx,
+			cluster, v1beta1.PGBackRestRepo{},
+			"",
+			nil, nil,
+		)
+
+		for _, container := range spec.Template.Spec.Containers {
+			assert.Assert(t, cmp.MarshalContains(container.VolumeMounts,
+				`
+- mountPath: /volumes/stuff
+  name: volumes-stuff`))
+		}
+
+		assert.Assert(t, cmp.MarshalContains(spec.Template.Spec.Volumes,
+			`
+- name: volumes-stuff
+  persistentVolumeClaim:
+    claimName: additional-pvc`))
+
+		// No events created
+		assert.Equal(t, len(recorder.Events), 0)
+	})
 }
 
 func TestGenerateRepoHostIntent(t *testing.T) {
@@ -3123,6 +3163,43 @@ func TestGenerateRepoHostIntent(t *testing.T) {
 		assert.NilError(t, err)
 		assert.Equal(t, *sts.Spec.Replicas, int32(0))
 	})
+
+	t.Run("AdditionalVolumes", func(t *testing.T) {
+		cluster := &v1beta1.PostgresCluster{
+			Spec: v1beta1.PostgresClusterSpec{
+				Backups: v1beta1.Backups{
+					PGBackRest: v1beta1.PGBackRestArchive{
+						RepoHost: &v1beta1.PGBackRestRepoHost{
+							Volumes: &v1beta1.PGBackRestVolumesSpec{
+								Additional: []v1beta1.AdditionalVolume{
+									{
+										ClaimName: "additional-pvc",
+										Name:      "stuff",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+		observed := &observedInstances{forCluster: []*Instance{{}}}
+		sts, err := r.generateRepoHostIntent(ctx, cluster, "", &RepoResources{}, observed, "")
+		assert.NilError(t, err)
+
+		for _, container := range sts.Spec.Template.Spec.Containers {
+			assert.Assert(t, cmp.MarshalContains(container.VolumeMounts,
+				`
+- mountPath: /volumes/stuff
+  name: volumes-stuff`))
+		}
+
+		assert.Assert(t, cmp.MarshalContains(sts.Spec.Template.Spec.Volumes,
+			`
+- name: volumes-stuff
+  persistentVolumeClaim:
+    claimName: additional-pvc`))
+	})
 }
 
 func TestGenerateRestoreJobIntent(t *testing.T) {
@@ -3174,6 +3251,14 @@ func TestGenerateRestoreJobIntent(t *testing.T) {
 			Operator: "Exist",
 		}},
 		PriorityClassName: initialize.String("some-priority-class"),
+		Volumes: &v1beta1.PGBackRestVolumesSpec{
+			Additional: []v1beta1.AdditionalVolume{
+				{
+					ClaimName: "additional-pvc",
+					Name:      "stuff",
+				},
+			},
+		},
 	}
 	cluster := &v1beta1.PostgresCluster{
 		ObjectMeta: metav1.ObjectMeta{
@@ -3266,6 +3351,9 @@ func TestGenerateRestoreJobIntent(t *testing.T) {
 								assert.DeepEqual(t, job.Spec.Template.Spec.Containers[0].VolumeMounts,
 									[]corev1.VolumeMount{{
 										Name: "mount",
+									}, {
+										Name:      "volumes-stuff",
+										MountPath: "/volumes/stuff",
 									}})
 							})
 							t.Run("Env", func(t *testing.T) {
@@ -3289,6 +3377,13 @@ func TestGenerateRestoreJobIntent(t *testing.T) {
 							assert.DeepEqual(t, job.Spec.Template.Spec.Volumes,
 								[]corev1.Volume{{
 									Name: "volume",
+								}, {
+									Name: "volumes-stuff",
+									VolumeSource: corev1.VolumeSource{
+										PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+											ClaimName: "additional-pvc",
+										},
+									},
 								}})
 						})
 						t.Run("Affinity", func(t *testing.T) {

internal/controller/postgrescluster/pgbackrest_test.go

@@ -311,6 +311,10 @@ type PostgresClusterDataSource struct {
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
+	// Volumes to add to Restore Job Pods
+	// +optional
+	Volumes *v1beta1.PGBackRestVolumesSpec `json:"volumes,omitempty"`
 }
 
 // Default defines several key default values for a Postgres cluster.

pkg/apis/postgres-operator.crunchydata.com/v1/postgrescluster_types.go

@@ -175,6 +175,21 @@ type BackupJobs struct {
 	// +optional
 	// +kubebuilder:validation:Minimum=60
 	TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"`
+
+	// Volumes to add to Backup Job Pods
+	// +optional
+	Volumes *PGBackRestVolumesSpec `json:"volumes,omitempty"`
+}
+
+// PGBackRestVolumesSpec defines the configuration for pgBackRest additional volumes
+type PGBackRestVolumesSpec struct {
+	// Additional pre-existing volumes to add to the pod.
+	// ---
+	// +optional
+	// +listType=map
+	// +listMapKey=name
+	// +kubebuilder:validation:MaxItems=10
+	Additional []AdditionalVolume `json:"additional,omitempty"`
 }
 
 // PGBackRestManualBackup contains information that is used for creating a
@@ -230,6 +245,10 @@ type PGBackRestRepoHost struct {
 	// Deprecated: Repository hosts use mTLS for encryption, authentication, and authorization.
 	// +optional
 	SSHSecret *corev1.SecretProjection `json:"sshSecret,omitempty"`
+
+	// Volumes to add to the Repo Host Pod
+	// +optional
+	Volumes *PGBackRestVolumesSpec `json:"volumes,omitempty"`
 }
 
 // PGBackRestRestore defines an in-place restore for the PostgresCluster.
@@ -459,4 +478,8 @@ type PGBackRestDataSource struct {
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
+	// Volumes to add to Restore Job Pods
+	// +optional
+	Volumes *PGBackRestVolumesSpec `json:"volumes,omitempty"`
 }

pkg/apis/postgres-operator.crunchydata.com/v1/zz_generated.deepcopy.go

@@ -308,6 +308,10 @@ type PostgresClusterDataSource struct {
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
+	// Volumes to add to Restore Job Pods
+	// +optional
+	Volumes *PGBackRestVolumesSpec `json:"volumes,omitempty"`
 }
 
 // Default defines several key default values for a Postgres cluster.