From 8919284188c46280ca3e1008faebef72ec368453 Mon Sep 17 00:00:00 2001 From: Jonathan Remy Date: Thu, 6 Nov 2025 08:04:20 +0100 Subject: [PATCH] feat(rdb): add benchmarks for instance get, backup get/list, and database list --- .../rdb/v1/custom_benchmark_test.go | 420 ++++++++++++++++++ internal/namespaces/rdb/v1/helper_test.go | 109 +++++ .../rdb/v1/testdata/benchmark.baseline | 10 + 3 files changed, 539 insertions(+) create mode 100644 internal/namespaces/rdb/v1/custom_benchmark_test.go create mode 100644 internal/namespaces/rdb/v1/testdata/benchmark.baseline diff --git a/internal/namespaces/rdb/v1/custom_benchmark_test.go b/internal/namespaces/rdb/v1/custom_benchmark_test.go new file mode 100644 index 0000000000..9ded21ec87 --- /dev/null +++ b/internal/namespaces/rdb/v1/custom_benchmark_test.go @@ -0,0 +1,420 @@ +package rdb_test + +import ( + "bytes" + "context" + "os" + "sort" + "testing" + "time" + + "github.com/scaleway/scaleway-cli/v2/core" + "github.com/scaleway/scaleway-cli/v2/internal/namespaces/rdb/v1" + rdbSDK "github.com/scaleway/scaleway-sdk-go/api/rdb/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// Benchmarks for RDB commands. +// +// Baseline stored in testdata/benchmark.baseline (like golden files). +// +// To compare performance: +// +// benchstat testdata/benchmark.baseline <(CLI_RUN_BENCHMARKS=true go test -bench=. -benchtime=100x .) +// +// To update baseline: +// +// CLI_RUN_BENCHMARKS=true go test -bench=. -benchtime=100x . > testdata/benchmark.baseline + +const ( + defaultCmdTimeout = 30 * time.Second + instanceReadyTimeout = 3 * time.Minute +) + +func setupBenchmark(b *testing.B) (*scw.Client, core.TestMetadata, func(args []string) any) { + b.Helper() + + clientOpts := []scw.ClientOption{ + scw.WithDefaultRegion(scw.RegionFrPar), + scw.WithDefaultZone(scw.ZoneFrPar1), + scw.WithUserAgent("cli-benchmark-test"), + scw.WithEnv(), + } + + config, err := scw.LoadConfig() + if err == nil { + activeProfile, err := config.GetActiveProfile() + if err == nil { + envProfile := scw.LoadEnvProfile() + profile := scw.MergeProfiles(activeProfile, envProfile) + clientOpts = append(clientOpts, scw.WithProfile(profile)) + } + } + + client, err := scw.NewClient(clientOpts...) + if err != nil { + b.Fatalf( + "Failed to create Scaleway client: %v\nMake sure you have configured your credentials with 'scw config'", + err, + ) + } + + meta := core.TestMetadata{ + "t": b, + } + + executeCmd := func(args []string) any { + stdoutBuffer := &bytes.Buffer{} + stderrBuffer := &bytes.Buffer{} + _, result, err := core.Bootstrap(&core.BootstrapConfig{ + Args: args, + Commands: rdb.GetCommands().Copy(), + BuildInfo: nil, + Stdout: stdoutBuffer, + Stderr: stderrBuffer, + Client: client, + DisableTelemetry: true, + DisableAliases: true, + OverrideEnv: map[string]string{}, + Ctx: context.Background(), + }) + if err != nil { + b.Errorf("error executing cmd (%s): %v\nstdout: %s\nstderr: %s", + args, err, stdoutBuffer.String(), stderrBuffer.String()) + } + + return result + } + + return client, meta, executeCmd +} + +func cleanupWithRetry(b *testing.B, name string, resourceID string, cleanupFn func() error) { + b.Helper() + + if err := cleanupFn(); err != nil { + b.Logf("cleanup failed (%s=%s): %v; retrying...", name, resourceID, err) + time.Sleep(2 * time.Second) + if err2 := cleanupFn(); err2 != nil { + b.Errorf("final cleanup failure (%s=%s): %v", name, resourceID, err2) + } + } +} + +type benchmarkStats struct { + timings []time.Duration + enabled bool +} + +func newBenchmarkStats() *benchmarkStats { + return &benchmarkStats{ + enabled: os.Getenv("CLI_BENCH_TRACE") == "true", + timings: make([]time.Duration, 0, 1000), + } +} + +func (s *benchmarkStats) record(d time.Duration) { + s.timings = append(s.timings, d) +} + +func (s *benchmarkStats) getMean() time.Duration { + if len(s.timings) == 0 { + return 0 + } + + var sum time.Duration + for _, t := range s.timings { + sum += t + } + + return sum / time.Duration(len(s.timings)) +} + +func (s *benchmarkStats) report(b *testing.B) { + b.Helper() + + if !s.enabled || len(s.timings) == 0 { + return + } + + sort.Slice(s.timings, func(i, j int) bool { + return s.timings[i] < s.timings[j] + }) + + minVal := s.timings[0] + maxVal := s.timings[len(s.timings)-1] + median := s.timings[len(s.timings)/2] + p95 := s.timings[int(float64(len(s.timings))*0.95)] + mean := s.getMean() + + b.Logf("Distribution (n=%d): min=%v median=%v mean=%v p95=%v max=%v", + len(s.timings), minVal, median, mean, p95, maxVal) +} + +func BenchmarkInstanceGet(b *testing.B) { + if os.Getenv("CLI_RUN_BENCHMARKS") != "true" { + b.Skip("Skipping benchmark. Set CLI_RUN_BENCHMARKS=true to run.") + } + + client, meta, executeCmd := setupBenchmark(b) + + ctx := &core.BeforeFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + err := createInstanceDirect(engine)(ctx) + if err != nil { + b.Fatalf("Failed to create instance: %v", err) + } + + instance := meta["Instance"].(rdb.CreateInstanceResult).Instance + + b.Cleanup(func() { + afterCtx := &core.AfterFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + cleanupWithRetry(b, "instance", instance.ID, func() error { + return deleteInstanceDirect()(afterCtx) + }) + }) + + stats := newBenchmarkStats() + b.ResetTimer() + b.ReportAllocs() + + for range b.N { + start := time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), defaultCmdTimeout) + done := make(chan any, 1) + + go func() { + done <- executeCmd([]string{"scw", "rdb", "instance", "get", instance.ID}) + }() + + select { + case <-done: + stats.record(time.Since(start)) + case <-ctx.Done(): + cancel() + b.Fatalf("command timeout after %v", defaultCmdTimeout) + } + cancel() + } + + b.StopTimer() + stats.report(b) +} + +func BenchmarkBackupGet(b *testing.B) { + if os.Getenv("CLI_RUN_BENCHMARKS") != "true" { + b.Skip("Skipping benchmark. Set CLI_RUN_BENCHMARKS=true to run.") + } + + client, meta, executeCmd := setupBenchmark(b) + + ctx := &core.BeforeFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + err := createInstanceDirect(engine)(ctx) + if err != nil { + b.Fatalf("Failed to create instance: %v", err) + } + + instance := meta["Instance"].(rdb.CreateInstanceResult).Instance + + if err := waitForInstanceReady(executeCmd, instance.ID, instanceReadyTimeout); err != nil { + b.Fatalf("Instance not ready: %v", err) + } + + err = createBackupDirect("Backup")(ctx) + if err != nil { + b.Fatalf("Failed to create backup: %v", err) + } + + backup := meta["Backup"].(*rdbSDK.DatabaseBackup) + + b.Cleanup(func() { + afterCtx := &core.AfterFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + cleanupWithRetry(b, "backup", backup.ID, func() error { + return deleteBackupDirect("Backup")(afterCtx) + }) + cleanupWithRetry(b, "instance", instance.ID, func() error { + return deleteInstanceDirect()(afterCtx) + }) + }) + + stats := newBenchmarkStats() + b.ResetTimer() + b.ReportAllocs() + + for range b.N { + start := time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), defaultCmdTimeout) + done := make(chan any, 1) + + go func() { + done <- executeCmd([]string{"scw", "rdb", "backup", "get", backup.ID}) + }() + + select { + case <-done: + stats.record(time.Since(start)) + case <-ctx.Done(): + cancel() + b.Fatalf("command timeout after %v", defaultCmdTimeout) + } + cancel() + } + + b.StopTimer() + stats.report(b) +} + +func BenchmarkBackupList(b *testing.B) { + if os.Getenv("CLI_RUN_BENCHMARKS") != "true" { + b.Skip("Skipping benchmark. Set CLI_RUN_BENCHMARKS=true to run.") + } + + client, meta, executeCmd := setupBenchmark(b) + + ctx := &core.BeforeFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + err := createInstanceDirect(engine)(ctx) + if err != nil { + b.Fatalf("Failed to create instance: %v", err) + } + + instance := meta["Instance"].(rdb.CreateInstanceResult).Instance + + if err := waitForInstanceReady(executeCmd, instance.ID, instanceReadyTimeout); err != nil { + b.Fatalf("Instance not ready: %v", err) + } + + err = createBackupDirect("Backup1")(ctx) + if err != nil { + b.Fatalf("Failed to create backup 1: %v", err) + } + err = createBackupDirect("Backup2")(ctx) + if err != nil { + b.Fatalf("Failed to create backup 2: %v", err) + } + + backup1 := meta["Backup1"].(*rdbSDK.DatabaseBackup) + backup2 := meta["Backup2"].(*rdbSDK.DatabaseBackup) + + b.Cleanup(func() { + afterCtx := &core.AfterFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + cleanupWithRetry(b, "backup1", backup1.ID, func() error { + return deleteBackupDirect("Backup1")(afterCtx) + }) + cleanupWithRetry(b, "backup2", backup2.ID, func() error { + return deleteBackupDirect("Backup2")(afterCtx) + }) + cleanupWithRetry(b, "instance", instance.ID, func() error { + return deleteInstanceDirect()(afterCtx) + }) + }) + + stats := newBenchmarkStats() + b.ResetTimer() + b.ReportAllocs() + + for range b.N { + start := time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), defaultCmdTimeout) + done := make(chan any, 1) + + go func() { + done <- executeCmd([]string{"scw", "rdb", "backup", "list", "instance-id=" + instance.ID}) + }() + + select { + case <-done: + stats.record(time.Since(start)) + case <-ctx.Done(): + cancel() + b.Fatalf("command timeout after %v", defaultCmdTimeout) + } + cancel() + } + + b.StopTimer() + stats.report(b) +} + +func BenchmarkDatabaseList(b *testing.B) { + if os.Getenv("CLI_RUN_BENCHMARKS") != "true" { + b.Skip("Skipping benchmark. Set CLI_RUN_BENCHMARKS=true to run.") + } + + client, meta, executeCmd := setupBenchmark(b) + + ctx := &core.BeforeFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + err := createInstanceDirect(engine)(ctx) + if err != nil { + b.Fatalf("Failed to create instance: %v", err) + } + + instance := meta["Instance"].(rdb.CreateInstanceResult).Instance + + b.Cleanup(func() { + afterCtx := &core.AfterFuncCtx{ + Client: client, + ExecuteCmd: executeCmd, + Meta: meta, + } + cleanupWithRetry(b, "instance", instance.ID, func() error { + return deleteInstanceDirect()(afterCtx) + }) + }) + + stats := newBenchmarkStats() + b.ResetTimer() + b.ReportAllocs() + + for range b.N { + start := time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), defaultCmdTimeout) + done := make(chan any, 1) + + go func() { + done <- executeCmd([]string{"scw", "rdb", "database", "list", "instance-id=" + instance.ID}) + }() + + select { + case <-done: + stats.record(time.Since(start)) + case <-ctx.Done(): + cancel() + b.Fatalf("command timeout after %v", defaultCmdTimeout) + } + cancel() + } + + b.StopTimer() + stats.report(b) +} diff --git a/internal/namespaces/rdb/v1/helper_test.go b/internal/namespaces/rdb/v1/helper_test.go index b1067f4886..6296726557 100644 --- a/internal/namespaces/rdb/v1/helper_test.go +++ b/internal/namespaces/rdb/v1/helper_test.go @@ -1,8 +1,10 @@ package rdb_test import ( + "context" "errors" "fmt" + "time" "github.com/scaleway/scaleway-cli/v2/core" "github.com/scaleway/scaleway-cli/v2/internal/namespaces/rdb/v1" @@ -113,3 +115,110 @@ func deleteInstance() core.AfterFunc { func deleteInstanceAndWait() core.AfterFunc { return core.ExecAfterCmd("scw rdb instance delete {{ .Instance.ID }} --wait") } + +func createInstanceDirect(_ string) core.BeforeFunc { + return func(ctx *core.BeforeFuncCtx) error { + result := ctx.ExecuteCmd([]string{ + "scw", "rdb", "instance", "create", + "node-type=DB-DEV-S", + "is-ha-cluster=false", + "name=" + name, + "engine=" + engine, + "user-name=" + user, + "password=" + password, + "--wait", + }) + ctx.Meta["Instance"] = result + + return nil + } +} + +func createBackupDirect(metaKey string) core.BeforeFunc { + return func(ctx *core.BeforeFuncCtx) error { + instanceResult := ctx.Meta["Instance"].(rdb.CreateInstanceResult) + instance := instanceResult.Instance + + result := ctx.ExecuteCmd([]string{ + "scw", "rdb", "backup", "create", + "name=cli-test-backup", + "expires-at=2032-01-02T15:04:05-07:00", + "instance-id=" + instance.ID, + "database-name=rdb", + "--wait", + }) + ctx.Meta[metaKey] = result + + return nil + } +} + +func deleteBackupDirect(metaKey string) core.AfterFunc { + return func(ctx *core.AfterFuncCtx) error { + backup := ctx.Meta[metaKey].(*rdbSDK.DatabaseBackup) + ctx.ExecuteCmd([]string{ + "scw", "rdb", "backup", "delete", + backup.ID, + }) + + return nil + } +} + +func deleteInstanceDirect() core.AfterFunc { + return func(ctx *core.AfterFuncCtx) error { + instance := ctx.Meta["Instance"].(rdb.CreateInstanceResult).Instance + ctx.ExecuteCmd([]string{ + "scw", "rdb", "instance", "delete", + instance.ID, + }) + + return nil + } +} + +func waitForInstanceReady( + executeCmd func([]string) any, + instanceID string, + maxWait time.Duration, +) error { + ctx, cancel := context.WithTimeout(context.Background(), maxWait) + defer cancel() + + backoff := time.Second + for { + select { + case <-ctx.Done(): + return fmt.Errorf( + "timeout waiting for instance %s to be ready for operations", + instanceID, + ) + default: + result := executeCmd([]string{"scw", "rdb", "instance", "get", instanceID}) + + // Try direct type assertion first + if instance, ok := result.(*rdbSDK.Instance); ok { + if instance.Status == rdbSDK.InstanceStatusReady { + time.Sleep(5 * time.Second) + + return nil + } + } else { + v := result.(struct { + *rdbSDK.Instance + ACLs []*rdbSDK.ACLRule `json:"acls"` + }) + if v.Instance != nil && v.Instance.Status == rdbSDK.InstanceStatusReady { + time.Sleep(5 * time.Second) + + return nil + } + } + + time.Sleep(backoff) + if backoff < 10*time.Second { + backoff *= 2 + } + } + } +} diff --git a/internal/namespaces/rdb/v1/testdata/benchmark.baseline b/internal/namespaces/rdb/v1/testdata/benchmark.baseline new file mode 100644 index 0000000000..4037c89950 --- /dev/null +++ b/internal/namespaces/rdb/v1/testdata/benchmark.baseline @@ -0,0 +1,10 @@ +goos: darwin +goarch: amd64 +pkg: github.com/scaleway/scaleway-cli/v2/internal/namespaces/rdb/v1 +cpu: VirtualApple @ 2.50GHz +BenchmarkInstanceGet-11 100 239648044 ns/op 350232 B/op 4121 allocs/op +BenchmarkBackupGet-11 100 85116384 ns/op 271182 B/op 2839 allocs/op +BenchmarkBackupList-11 100 118312364 ns/op 284095 B/op 2996 allocs/op +BenchmarkDatabaseList-11 100 136427978 ns/op 283705 B/op 3046 allocs/op +PASS +