roachprod-microbench: post GitHub issues for performance regressions

Previously, performance regressions detected during the weekly microbenchmark
comparison were only reported via Slack notifications. This made it difficult
to track and ensure timely follow-up on regressions, as they were often
discussed informally without formal issue tracking.

This change extends the existing `--post-issues` flag to work with the compare
command. When enabled, the system automatically creates GitHub issues for
performance regressions that exceed 20% (the "red" regression threshold). Each
issue includes:
- Package name and list of regressed benchmarks
- Regression percentages and formatted deltas
- Link to the Google Sheet with detailed comparison data
- Labels: O-microbench and C-performance for easy filtering

The implementation reuses the same GitHub posting infrastructure and
environment variables (GITHUB_BRANCH, GITHUB_SHA, GITHUB_BINARY) as the
existing benchmark failure reporting. Issues are created per package to avoid
spam, with up to 10 regressions listed in each issue summary.

This change also renames `postBenchmarkIssue` to `postIssuesToGitHub` for
consistency, as it now handles both execution failures and performance
regressions.

Epic: None
Release note: None

Author: ibreakthecloud

build/teamcity/cockroach/nightlies/microbenchmark_weekly.sh

@@ -207,6 +207,7 @@ if [ -d "$output_dir/experiment" ] && [ "$(ls -A "$output_dir/experiment")" ] \
     --sheet-desc="$sheet_description" \
     ${influx_token:+--influx-token="$influx_token"} \
     ${influx_host:+--influx-host="$influx_host"} \
+    ${TRIGGERED_BUILD:+--post-issues} \
     2>&1 | tee "$output_dir/sheets.txt"
 else
   echo "No microbenchmarks were run. Skipping comparison."

pkg/cmd/roachprod-microbench/compare.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench/google"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench/model"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench/util"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/errors/oserror"
@@ -37,6 +38,7 @@ type compareConfig struct {
 	baselineDir   string
 	sheetDesc     string
 	threshold     float64
+	postIssues    bool
 }
 
 type slackConfig struct {
@@ -349,6 +351,52 @@ func (c *compare) compareUsingThreshold(comparisonResultsMap model.ComparisonRes
 	return nil
 }
 
+func (c *compare) postRegressionIssues(
+	links map[string]string, comparisonResultsMap model.ComparisonResultsMap,
+) error {
+	loggerCfg := logger.Config{Stdout: os.Stdout, Stderr: os.Stderr}
+	l, err := loggerCfg.NewLogger("")
+	if err != nil {
+		return errors.Wrap(err, "failed to create logger for posting issues")
+	}
+
+	for pkgName, comparisonResults := range comparisonResultsMap {
+		var regressions []regressionInfo
+
+		for _, result := range comparisonResults {
+			for _, detail := range result.Comparisons {
+				comparison := detail.Comparison
+				metric := result.Metric
+
+				// Check if this is a regression
+				isRegression := (comparison.Delta < 0 && metric.Better > 0) ||
+					(comparison.Delta > 0 && metric.Better < 0)
+
+				if isRegression && math.Abs(comparison.Delta) >= slackPercentageThreshold {
+					regressions = append(regressions, regressionInfo{
+						benchmarkName:  detail.BenchmarkName,
+						metricUnit:     result.Metric.Unit,
+						percentChange:  math.Abs(comparison.Delta),
+						formattedDelta: comparison.FormattedDelta,
+					})
+				}
+			}
+		}
+
+		if len(regressions) > 0 {
+			sheetLink := links[pkgName]
+			formatter, req := createRegressionPostRequest(pkgName, regressions, sheetLink, c.sheetDesc)
+			err := postIssuesToGitHub(c.ctx, l, formatter, req)
+			if err != nil {
+				return errors.Wrapf(err, "failed to post regression issue for package %s", pkgName)
+			}
+			log.Printf("Posted regression issue for package: %s with %d regression(s)", pkgName, len(regressions))
+		}
+	}
+
+	return nil
+}
+
 func (c *compare) pushToInfluxDB(comparisonResultsMap model.ComparisonResultsMap) error {
 	client := influxdb2.NewClient(c.influxConfig.host, c.influxConfig.token)
 	defer client.Close()

pkg/cmd/roachprod-microbench/executor.go

@@ -460,7 +460,7 @@ func (e *executor) executeBenchmarks() error {
 			if e.postIssues && benchmarkResponse.key == e.postConfig.binary {
 				artifactsDir := fmt.Sprintf("%s/%s", e.outputDir, benchmarkResponse.key)
 				formatter, req := createBenchmarkPostRequest(artifactsDir, response, timeout)
-				err = postBenchmarkIssue(context.Background(), e.log, formatter, req)
+				err = postIssuesToGitHub(context.Background(), e.log, formatter, req)
 				if err != nil {
 					e.log.Errorf("Failed to post benchmark issue - %v", err)
 					executorError = errors.CombineErrors(executorError, errors.Wrap(err, "failed to post benchmark issue"))

pkg/cmd/roachprod-microbench/github.go

@@ -43,8 +43,50 @@ func createBenchmarkPostRequest(
 	return formatter, req
 }
 
-// postBenchmarkIssue posts a benchmark issue to github.
-func postBenchmarkIssue(
+// regressionInfo holds information about a single benchmark regression.
+type regressionInfo struct {
+	benchmarkName  string
+	metricUnit     string
+	percentChange  float64
+	formattedDelta string
+}
+
+// createRegressionPostRequest creates a post request for benchmark performance regressions.
+func createRegressionPostRequest(
+	pkgName string, regressions []regressionInfo, sheetLink, description string,
+) (issues.IssueFormatter, issues.PostRequest) {
+	// Build the regression summary message
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("Performance regressions detected in package %s\n\n", pkgName))
+	sb.WriteString(fmt.Sprintf("Comparison: %s\n\n", description))
+	sb.WriteString(fmt.Sprintf("Found %d benchmark(s) with regressions ≥20%%:\n\n", len(regressions)))
+
+	for i, reg := range regressions {
+		if i >= 10 { // Limit to 10 regressions in the summary
+			sb.WriteString(fmt.Sprintf("... and %d more regression(s)\n", len(regressions)-i))
+			break
+		}
+		sb.WriteString(fmt.Sprintf("• %s (%s): %s (%.1f%%)\n",
+			reg.benchmarkName, reg.metricUnit, reg.formattedDelta, reg.percentChange))
+	}
+
+	sb.WriteString(fmt.Sprintf("\nDetailed comparison: %s\n", sheetLink))
+
+	// Create a failure object for the package with all regressions
+	title := fmt.Sprintf("%s: performance regression", pkgName)
+	f := githubpost.MicrobenchmarkFailure(
+		pkgName,
+		title,
+		sb.String(),
+	)
+
+	formatter, req := githubpost.DefaultFormatter(context.Background(), f)
+	req.Labels = append(req.Labels, "O-microbench", "C-performance")
+	return formatter, req
+}
+
+// postIssuesToGitHub posts a benchmark issue to github.
+func postIssuesToGitHub(
 	ctx context.Context, l *logger.Logger, formatter issues.IssueFormatter, req issues.PostRequest,
 ) error {
 	opts := issues.DefaultOptionsFromEnv()

pkg/cmd/roachprod-microbench/main.go

@@ -173,6 +173,12 @@ func makeCompareCommand() *cobra.Command {
 					return err
 				}
 			}
+			if c.postIssues {
+				err = c.postRegressionIssues(links, comparisonResult)
+				if err != nil {
+					return err
+				}
+			}
 		}
 
 		if c.influxConfig.token != "" {
@@ -207,6 +213,7 @@ func makeCompareCommand() *cobra.Command {
 	cmd.Flags().StringVar(&config.influxConfig.token, "influx-token", config.influxConfig.token, "pass an InfluxDB auth token to push the results to InfluxDB")
 	cmd.Flags().StringToStringVar(&config.influxConfig.metadata, "influx-metadata", config.influxConfig.metadata, "pass metadata to add to the InfluxDB measurement")
 	cmd.Flags().Float64Var(&config.threshold, "threshold", config.threshold, "threshold in percentage value for detecting perf regression ")
+	cmd.Flags().BoolVar(&config.postIssues, "post-issues", config.postIssues, "post GitHub issues for performance regressions (requires env vars for github issues to be set)")
 	return cmd
 }