spanconfigjob: include underlying error in job failure message

Previously, when the span config reconciliation job exhausted its
internal retries, it would fail with a generic "reconciliation
unsuccessful, failing job" message without including the underlying
cause of the failure.

This change tracks the last reconciliation error and wraps it in the
final job error message.

Closes #155380

Epic: None
Release note: none

Author: spilchen

pkg/spanconfig/spanconfigjob/job.go

@@ -134,6 +134,7 @@ func (r *resumer) Resume(ctx context.Context, execCtxI interface{}) (jobErr erro
 	}
 
 	var lastCheckpoint = hlc.Timestamp{}
+	var lastErr error
 	const aWhile = 5 * time.Minute // arbitrary but much longer than a retry
 	for retrier := retry.StartWithCtx(ctx, retryOpts); retrier.Next(); {
 		started := timeutil.Now()
@@ -167,6 +168,7 @@ func (r *resumer) Resume(ctx context.Context, execCtxI interface{}) (jobErr erro
 				Checkpoint: rc.Checkpoint(),
 			})
 		}); err != nil {
+			lastErr = err
 			if shouldSkipRetry {
 				break
 			}
@@ -192,6 +194,9 @@ func (r *resumer) Resume(ctx context.Context, execCtxI interface{}) (jobErr erro
 		return nil // we're done here (the stopper was stopped, Reconcile exited cleanly)
 	}
 
+	if lastErr != nil {
+		return errors.Wrap(lastErr, "reconciliation unsuccessful, failing job")
+	}
 	return errors.Newf("reconciliation unsuccessful, failing job")
 }
 

pkg/spanconfig/spanconfigmanager/BUILD.bazel

@@ -43,6 +43,7 @@ go_test(
         "//pkg/sql/catalog",
         "//pkg/sql/isql",
         "//pkg/testutils",
+        "//pkg/testutils/jobutils",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/sqlutils",
         "//pkg/testutils/testcluster",

pkg/spanconfig/spanconfigmanager/manager_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
 	"github.com/cockroachdb/cockroach/pkg/sql/isql"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/jobutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
@@ -365,6 +366,66 @@ func TestReconciliationJobErrorAndRecovery(t *testing.T) {
 	mu.Unlock()
 }
 
+// TestReconciliationJobFinalErrorIncludesCause verifies that the reconciliation
+// job surfaces the last reconciliation error when internal retries are exhausted.
+func TestReconciliationJobFinalErrorIncludesCause(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	ctx := context.Background()
+	const injectedErrMsg = "injected reconciliation failure"
+	injectedErr := errors.New(injectedErrMsg)
+
+	var jobID jobspb.JobID
+	srv, db, _ := serverutils.StartServer(t, base.TestServerArgs{
+		Knobs: base.TestingKnobs{
+			SpanConfig: &spanconfig.TestingKnobs{
+				ManagerDisableJobCreation: true, // disable automatic job creation; we'll create one explicitly
+				JobDisableInternalRetry:   true,
+				JobOnCheckpointInterceptor: func(_ hlc.Timestamp) error {
+					return injectedErr
+				},
+			},
+		},
+	})
+	defer srv.Stopper().Stop(ctx)
+	ts := srv.ApplicationLayer()
+
+	tdb := sqlutils.MakeSQLRunner(db)
+
+	manager := spanconfigmanager.New(
+		ts.InternalDB().(isql.DB),
+		ts.JobRegistry().(*jobs.Registry),
+		ts.AppStopper(),
+		ts.ClusterSettings(),
+		ts.SpanConfigReconciler().(spanconfig.Reconciler),
+		&spanconfig.TestingKnobs{
+			ManagerCreatedJobInterceptor: func(jobI interface{}) {
+				jobID = jobI.(*jobs.Job).ID()
+			},
+		},
+	)
+
+	started, err := manager.TestingCreateAndStartJobIfNoneExists(ctx, ts.ClusterSettings())
+	require.NoError(t, err)
+	require.True(t, started)
+
+	testutils.SucceedsSoon(t, func() error {
+		if jobID == 0 {
+			return errors.New("waiting for job to be created")
+		}
+		return nil
+	})
+
+	waitForJobState(t, tdb, jobID, jobs.StateFailed)
+
+	payload := jobutils.GetJobPayload(t, tdb, jobID)
+	require.NotNil(t, payload.FinalResumeError)
+	decodedErr := errors.DecodeError(ctx, *payload.FinalResumeError)
+	require.ErrorContains(t, decodedErr, "reconciliation unsuccessful, failing job")
+	require.ErrorContains(t, decodedErr, injectedErrMsg)
+}
+
 // TestReconciliationUsesRightCheckpoint verifies that the reconciliation
 // internal retry uses the right checkpoint during internal retries.
 func TestReconciliationUsesRightCheckpoint(t *testing.T) {