roachtest: add github issue posting dry run mode

We already have the functionality in datadriven tests
to render github issue post requests as clickable links.
This change extends that functionality to roachtests in
the form of a --dry-run-issue-posting flag. This allows
for viewing what the output of a failed roachtest would
look like even if github posting is disabled.

Author: DarrylWong

pkg/cmd/roachtest/github.go

@@ -8,7 +8,9 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/bazci/githubpost/issues"
@@ -35,6 +37,7 @@ type GithubPoster interface {
 // githubIssues struct implements GithubPoster
 type githubIssues struct {
 	disable     bool
+	dryRun      bool
 	issuePoster func(context.Context, issues.Logger, issues.IssueFormatter, issues.PostRequest,
 		*issues.Options) (*issues.TestFailureIssue, error)
 	teamLoader func() (team.Map, error)
@@ -327,6 +330,10 @@ func (g *githubIssues) MaybePost(
 	message string,
 	params map[string]string,
 ) (*issues.TestFailureIssue, error) {
+	if g.dryRun {
+		return nil, g.dryRunPost(t, issueInfo, l, message, params)
+	}
+
 	skipReason := g.shouldPost(t)
 	if skipReason != "" {
 		l.Printf("skipping GitHub issue posting (%s)", skipReason)
@@ -352,3 +359,75 @@ func (g *githubIssues) MaybePost(
 		opts,
 	)
 }
+
+// dryRunPost simulates posting an issue to GitHub by rendering
+// the issue and logging a clickable link.
+func (g *githubIssues) dryRunPost(
+	t *testImpl,
+	issueInfo *githubIssueInfo,
+	l *logger.Logger,
+	message string,
+	params map[string]string,
+) error {
+	postRequest, err := g.createPostRequest(
+		t.Name(), t.start, t.end, t.spec, t.failures(),
+		message,
+		roachtestutil.UsingRuntimeAssertions(t), t.goCoverEnabled, params, issueInfo,
+	)
+	if err != nil {
+		return err
+	}
+	_, url, err := formatPostRequest(postRequest)
+	if err != nil {
+		return err
+	}
+	l.Printf("GitHub issue posting in dry-run mode:\n%s", url)
+	return nil
+}
+
+// formatPostRequest returns a string representation of the rendered PostRequest
+// as well as a link that can be followed to open the issue in Github. The rendered
+// PostRequest also includes the labels that would be applied to the issue as part
+// of the body.
+func formatPostRequest(req issues.PostRequest) (string, string, error) {
+	data := issues.TemplateData{
+		PostRequest:      req,
+		Parameters:       req.ExtraParams,
+		CondensedMessage: issues.CondensedMessage(req.Message),
+		Branch:           "test_branch",
+		Commit:           "test_SHA",
+		PackageNameShort: strings.TrimPrefix(req.PackageName, issues.CockroachPkgPrefix),
+	}
+
+	formatter := issues.UnitTestFormatter
+	r := &issues.Renderer{}
+	if err := formatter.Body(r, data); err != nil {
+		return "", "", err
+	}
+
+	var post strings.Builder
+	post.WriteString(r.String())
+
+	// Github labels are normally not part of the rendered issue body, but we want to
+	// still test that they are correctly set so append them here.
+	post.WriteString("\n------\nLabels:\n")
+	for _, label := range req.Labels {
+		post.WriteString(fmt.Sprintf("- <code>%s</code>\n", label))
+	}
+
+	u, err := url.Parse("https://github.com/cockroachdb/cockroach/issues/new")
+	if err != nil {
+		return "", "", err
+	}
+	q := u.Query()
+	q.Add("title", formatter.Title(data))
+	q.Add("body", post.String())
+	// Adding a template parameter is required to be able to view the rendered
+	// template on GitHub, otherwise it just takes you to the template selection
+	// page.
+	q.Add("template", "none")
+	u.RawQuery = q.Encode()
+	post.WriteString(fmt.Sprintf("Rendered:\n%s", u.String()))
+
+	return post.String(), u.String(), nil
+}

pkg/cmd/roachtest/github_test.go

@@ -7,7 +7,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"net/url"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -90,7 +89,7 @@ func TestShouldPost(t *testing.T) {
 
 		ti := &testImpl{spec: testSpec}
 		ti.mu.failures = c.failures
-		github := &githubIssues{disable: c.disableIssues}
+		github := &githubIssues{disable: c.disableIssues, dryRun: false}
 
 		skipReason := github.shouldPost(ti)
 		require.Equal(t, c.expectedReason, skipReason)
@@ -184,7 +183,7 @@ func TestCreatePostRequest(t *testing.T) {
 				}
 				require.NoError(t, err)
 
-				post, err := formatPostRequest(req)
+				post, _, err := formatPostRequest(req)
 				require.NoError(t, err)
 
 				return post
@@ -258,49 +257,3 @@ F250826 19:49:07.194443 3106 sql/sem/builtins/builtins.go:6063 ⋮ [T1,Vsystem,n
 		})
 	})
 }
-
-// formatPostRequest returns a string representation of the rendered PostRequest.
-// Additionally, it also includes labels, as well as a link that can be followed
-// to open the issue in Github.
-func formatPostRequest(req issues.PostRequest) (string, error) {
-	data := issues.TemplateData{
-		PostRequest:      req,
-		Parameters:       req.ExtraParams,
-		CondensedMessage: issues.CondensedMessage(req.Message),
-		Branch:           "test_branch",
-		Commit:           "test_SHA",
-		PackageNameShort: strings.TrimPrefix(req.PackageName, issues.CockroachPkgPrefix),
-	}
-
-	formatter := issues.UnitTestFormatter
-	r := &issues.Renderer{}
-	if err := formatter.Body(r, data); err != nil {
-		return "", err
-	}
-
-	var post strings.Builder
-	post.WriteString(r.String())
-
-	// Github labels are normally not part of the rendered issue body, but we want to
-	// still test that they are correctly set so append them here.
-	post.WriteString("\n------\nLabels:\n")
-	for _, label := range req.Labels {
-		post.WriteString(fmt.Sprintf("- <code>%s</code>\n", label))
-	}
-
-	u, err := url.Parse("https://github.com/cockroachdb/cockroach/issues/new")
-	if err != nil {
-		return "", err
-	}
-	q := u.Query()
-	q.Add("title", formatter.Title(data))
-	q.Add("body", post.String())
-	// Adding a template parameter is required to be able to view the rendered
-	// template on GitHub, otherwise it just takes you to the template selection
-	// page.
-	q.Add("template", "none")
-	u.RawQuery = q.Encode()
-	post.WriteString(fmt.Sprintf("Rendered:\n%s", u.String()))
-
-	return post.String(), nil
-}

pkg/cmd/roachtest/roachtestflags/flags.go

@@ -446,6 +446,12 @@ var (
 		Usage: `Disable posting GitHub issue for failures`,
 	})
 
+	DryRunIssuePosting bool
+	_                  = registerRunFlag(&DryRunIssuePosting, FlagInfo{
+		Name:  "dry-run-issue-posting",
+		Usage: `Enable dry-run mode for GitHub issue posting (formats issues but doesn't post them)`,
+	})
+
 	PromPort int = 2113
 	_            = registerRunFlag(&PromPort, FlagInfo{
 		Name: "prom-port",

pkg/cmd/roachtest/run.go

@@ -143,6 +143,7 @@ func runTests(register func(registry.Registry), filter *registry.TestFilter) err
 
 	github := &githubIssues{
 		disable:     runner.config.disableIssue,
+		dryRun:      runner.config.dryRunIssuePosting,
 		issuePoster: issues.Post,
 		teamLoader:  team.DefaultLoadTeams,
 	}

pkg/cmd/roachtest/test_runner.go

@@ -148,6 +148,8 @@ type testRunner struct {
 		skipClusterWipeOnAttach bool
 		// disableIssue disables posting GitHub issues for test failures.
 		disableIssue bool
+		// dryRunIssuePosting enables dry-run mode for GitHub issue posting.
+		dryRunIssuePosting bool
 		// overrideShutdownPromScrapeInterval overrides the default time a test runner waits to
 		// shut down, normally used to ensure a remote prometheus server has scraped the roachtest
 		// endpoint.
@@ -215,6 +217,7 @@ func newTestRunner(cr *clusterRegistry, stopper *stop.Stopper) *testRunner {
 	}
 	r.config.skipClusterWipeOnAttach = !roachtestflags.ClusterWipe
 	r.config.disableIssue = roachtestflags.DisableIssue
+	r.config.dryRunIssuePosting = roachtestflags.DryRunIssuePosting
 	r.workersMu.workers = make(map[string]*workerStatus)
 	return r
 }

pkg/cmd/roachtest/test_test.go

@@ -103,6 +103,7 @@ func defaultLoggingOpt(buf *syncedBuffer) loggingOpt {
 func defaultGithub(disable bool) GithubPoster {
 	return &githubIssues{
 		disable: disable,
+		dryRun:  false,
 		// issuePoster isn't mocked because an env var check exits MaybePost when
 		// the GitHub API isn't present, so technically setting it here doesn't
 		// matter