roachtest: plumb AOST used in backup fixture fingerprint to restore

In the backup fixture roachtest, an incremental backup can be taken after the
fingerprint aost is chosen. In this case, the restore roachtest should restore
at that system time, instead of from LATEST, else the restore roachtest could
mistakenly fail on a fingerprint violation.

This patch plumbs the backup fixture aost to the restore roachtest to avoid
this test flake.

Informs #152721
Informs #152724

Release note: none

Author: msbutler

pkg/cmd/roachtest/tests/backup_fixtures.go

@@ -445,17 +445,19 @@ func (bd *backupDriver) queryJobStates(
 // fingerprintFixture computes fingerprints for the fixture as of the time of
 // its last incremental backup. It maps the fully qualified name of each table
 // to its fingerprint.
-func (bd *backupDriver) fingerprintFixture(ctx context.Context) map[string]string {
+func (bd *backupDriver) fingerprintFixture(ctx context.Context, asOfTime string) map[string]string {
 	conn := bd.c.Conn(ctx, bd.t.L(), 1)
 	defer conn.Close()
 	return fingerprintDatabase(
-		bd.t, conn, bd.sp.fixture.DatabaseName(), bd.getLatestAOST(sqlutils.MakeSQLRunner(conn)),
-	)
+		bd.t, conn, bd.sp.fixture.DatabaseName(), asOfTime)
 }
 
 // getLatestAOST returns the end time as seen in SHOW BACKUP of the latest
 // backup in the fixture.
-func (bd *backupDriver) getLatestAOST(sql *sqlutils.SQLRunner) string {
+func (bd *backupDriver) getLatestAOST(ctx context.Context) string {
+	conn := bd.c.Conn(ctx, bd.t.L(), 1)
+	defer conn.Close()
+	sql := sqlutils.MakeSQLRunner(conn)
 	uri := bd.registry.URI(bd.fixture.DataPath)
 	query := fmt.Sprintf(
 		`SELECT end_time FROM
@@ -675,8 +677,9 @@ func registerBackupFixtures(r registry.Registry) {
 				stopWorkload()
 
 				if !bf.skipFingerprint {
-					fingerprint := bd.fingerprintFixture(ctx)
-					require.NoError(t, handle.SetFingerprint(ctx, fingerprint))
+					fingerprintTime := bd.getLatestAOST(ctx)
+					fingerprint := bd.fingerprintFixture(ctx, fingerprintTime)
+					require.NoError(t, handle.SetFingerprint(ctx, fingerprint, fingerprintTime))
 				}
 
 				require.NoError(t, handle.SetReadyAt(ctx))

pkg/cmd/roachtest/tests/restore.go

@@ -825,6 +825,11 @@ func (rd *restoreDriver) prepareCluster(ctx context.Context) {
 
 // getAOST gets the AOST to use in the restore cmd.
 func (rd *restoreDriver) getAOST(ctx context.Context) {
+	if rd.fixtureMetadata.FingerprintTime != "" {
+		rd.aost = rd.fixtureMetadata.FingerprintTime
+		rd.t.L().Printf("using AOST from fixture metadata: %s", rd.aost)
+		return
+	}
 	if !rd.sp.fullBackupOnly {
 		rd.aost = ""
 		return

pkg/roachprod/blobfixture/metadata.go

@@ -40,6 +40,9 @@ type FixtureMetadata struct {
 	// Note: this may not be present if the fixture was too large to be
 	// fingerprinted.
 	Fingerprint map[string]string `json:"fingerprint,omitempty"`
+
+	// FingerprintTime is the aost used by the fingerprint command.
+	FingerprintTime string `json:"fingerprint_time,omitempty"`
 }
 
 func (f *FixtureMetadata) MarshalJson() ([]byte, error) {

pkg/roachprod/blobfixture/registry.go

@@ -333,11 +333,14 @@ func (s *ScratchHandle) SetReadyAt(ctx context.Context) error {
 }
 
 // SetFingerprint sets the fingerprint for the fixture.
-func (s *ScratchHandle) SetFingerprint(ctx context.Context, fingerprint map[string]string) error {
+func (s *ScratchHandle) SetFingerprint(
+	ctx context.Context, fingerprint map[string]string, asOf string,
+) error {
 	s.metadata.Fingerprint = fingerprint
+	s.metadata.FingerprintTime = asOf
 	if err := s.registry.upsertMetadata(s.metadata); err != nil {
 		return err
 	}
-	s.logger.Printf("fixture '%s' fingerprint set to '%s'", s.metadata.DataPath, s.metadata.Fingerprint)
+	s.logger.Printf("fixture '%s' fingerprint set to '%s' with asOf time %s ", s.metadata.DataPath, s.metadata.Fingerprint, s.metadata.FingerprintTime)
 	return nil
 }