feat: add configurable initial delay with jitter to CAS backend validation

Add InitialDelay field to CASBackendCheckerOpts to allow configuring the initial delay before first validation. This prevents thundering herd issues when multiple pods start simultaneously.

Implementation:
- CAS backend checker now accepts InitialDelay via options
- Default behavior: 1 minute base delay + 0-5 minutes jitter (calculated in main.go)
- Changed OnlyDefaults from bool to *bool to distinguish between explicit false and default true
- Validation runs after initial delay, then continues with periodic checks

This ensures validation happens in background without affecting boot performance and spreads validation load across pods during rolling deployments.

Signed-off-by: Miguel Martinez <miguel@chainloop.dev>

Author: migmartri

app/controlplane/cmd/main.go

@@ -1,5 +1,5 @@
 //
-// Copyright 2024 The Chainloop Authors.
+// Copyright 2024-2025 The Chainloop Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	_ "net/http/pprof"
 	"os"
 	"time"
@@ -169,7 +170,20 @@ func main() {
 
 	// Start the background CAS Backend checker
 	if app.casBackendChecker != nil {
-		go app.casBackendChecker.Start(ctx, &biz.CASBackendCheckerOpts{CheckInterval: 30 * time.Minute})
+		// Calculate initial delay: 1 minute base + 0-5 minutes jitter
+		// This protects boot phase and spreads validation across pods
+		baseDelay := 1 * time.Minute
+		// #nosec G404 - using math/rand for jitter is acceptable, cryptographic randomness not required
+		jitter := time.Duration(rand.Intn(5*60)) * time.Second
+		initialDelay := baseDelay + jitter
+
+		// check all backends
+		var onlyDefaults = false
+		go app.casBackendChecker.Start(ctx, &biz.CASBackendCheckerOpts{
+			CheckInterval: 30 * time.Minute,
+			InitialDelay:  initialDelay,
+			OnlyDefaults:  &onlyDefaults,
+		})
 	}
 
 	// start and wait for stop signal

app/controlplane/pkg/biz/casbackend_checker.go

@@ -39,11 +39,13 @@ type CASBackendChecker struct {
 
 type CASBackendCheckerOpts struct {
 	// Whether to check only default backends or all backends
-	OnlyDefaults bool
+	OnlyDefaults *bool
 	// Interval between checks, defaults to 30 minutes
 	CheckInterval time.Duration
 	// Timeout for each individual backend validation, defaults to 10 seconds
 	ValidationTimeout time.Duration
+	// Initial delay before first validation (includes jitter). If not set, runs immediately.
+	InitialDelay time.Duration
 }
 
 // NewCASBackendChecker creates a new CAS backend checker that will periodically validate
@@ -65,28 +67,45 @@ func (c *CASBackendChecker) Start(ctx context.Context, opts *CASBackendCheckerOp
 	}
 
 	onlyDefaults := true
-	if opts != nil {
-		onlyDefaults = opts.OnlyDefaults
+	if opts != nil && opts.OnlyDefaults != nil {
+		onlyDefaults = *opts.OnlyDefaults
 	}
 
 	// Apply validation timeout from options if provided
 	if opts != nil && opts.ValidationTimeout > 0 {
 		c.validationTimeout = opts.ValidationTimeout
 	}
 
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
-
-	// Run one check immediately
-	if err := c.CheckAllBackends(ctx, onlyDefaults); err != nil {
-		c.logger.Errorf("initial CAS backend check failed: %v", err)
+	// Apply initial delay from options if provided
+	var initialDelay = 0 * time.Second
+	if opts != nil && opts.InitialDelay > 0 {
+		initialDelay = opts.InitialDelay
 	}
 
-	c.logger.Infof("CAS backend checker started with interval %s, checking %s, timeout %s",
+	// Wait for initial delay before starting if configured
+	c.logger.Infof("CAS backend checker will start in %s, then run every %s, checking %s, timeout %s",
+		initialDelay,
 		interval,
 		conditionalString(onlyDefaults, "only default backends", "all backends"),
 		c.validationTimeout)
 
+	select {
+	case <-ctx.Done():
+		c.logger.Info("CAS backend checker stopping due to context cancellation before initial check")
+		return
+	case <-time.After(initialDelay):
+		// Continue to first check
+	}
+
+	// Run first check
+	if err := c.CheckAllBackends(ctx, onlyDefaults); err != nil {
+		c.logger.Errorf("initial CAS backend check failed: %v", err)
+	}
+
+	// Start periodic checks
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-ctx.Done():