From f2ac98add44ff6cd3191e06aab69105ce87c1cba Mon Sep 17 00:00:00 2001 From: manab-pr Date: Fri, 17 Oct 2025 20:27:22 +0530 Subject: [PATCH] fix: make minimum step duration configurable for eBPF profiling This change addresses issue #4524 where users setting `collect_interval` below 15 seconds (e.g., 5s for eBPF profiling) were seeing their data displayed at 15-second intervals in the UI due to a hardcoded minimum step duration. Changes: - Added `--querier.min-step-duration` flag (default: 15s) to allow users to configure the minimum step duration for timeline calculations - Created `CalcPointIntervalWithMinInterval` function that accepts a custom minimum interval parameter - Updated HTTP handlers to pass the configurable value through the call chain - Added comprehensive tests covering eBPF use cases (1s, 5s intervals) - Maintains backward compatibility with 15-second default Users can now run Pyroscope with `--querier.min-step-duration=5s` to support fast eBPF profiling collection intervals while maintaining fine-grained resolution in the UI. Fixes #4524 --- pkg/api/api.go | 5 ++-- pkg/pyroscope/modules.go | 2 +- pkg/pyroscope/modules_experimental.go | 6 ++--- pkg/querier/http.go | 13 ++++++--- pkg/querier/querier.go | 6 +++-- pkg/querier/timeline/calculator.go | 15 +++++++++-- pkg/querier/timeline/calculator_test.go | 36 +++++++++++++++++++++++++ 7 files changed, 69 insertions(+), 14 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index b739c73a37..d6cbcd7605 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -10,6 +10,7 @@ import ( "flag" "fmt" "net/http" + "time" "connectrpc.com/connect" @@ -232,8 +233,8 @@ func (a *API) RegisterFeatureFlagsServiceHandler(svc capabilitiesv1connect.Featu capabilitiesv1connect.RegisterFeatureFlagsServiceHandler(a.server.HTTP, svc, a.connectOptionsAuthLogRecovery()...) } -func (a *API) RegisterPyroscopeHandlers(client querierv1connect.QuerierServiceClient) { - handlers := querier.NewHTTPHandlers(client) +func (a *API) RegisterPyroscopeHandlers(client querierv1connect.QuerierServiceClient, minStepDuration time.Duration) { + handlers := querier.NewHTTPHandlers(client, minStepDuration) a.RegisterRoute("/pyroscope/render", http.HandlerFunc(handlers.Render), a.registerOptionsReadPath()...) a.RegisterRoute("/pyroscope/render-diff", http.HandlerFunc(handlers.RenderDiff), a.registerOptionsReadPath()...) a.RegisterRoute("/pyroscope/label-values", http.HandlerFunc(handlers.LabelValues), a.registerOptionsReadPath()...) diff --git a/pkg/pyroscope/modules.go b/pkg/pyroscope/modules.go index 91f9563259..0dd7dad0e6 100644 --- a/pkg/pyroscope/modules.go +++ b/pkg/pyroscope/modules.go @@ -262,7 +262,7 @@ func (f *Pyroscope) initQuerier() (services.Service, error) { } if !f.isModuleActive(QueryFrontend) { - f.API.RegisterPyroscopeHandlers(querierSvc) + f.API.RegisterPyroscopeHandlers(querierSvc, f.Cfg.Querier.MinStepDuration) f.API.RegisterQuerierServiceHandler(querierSvc) } diff --git a/pkg/pyroscope/modules_experimental.go b/pkg/pyroscope/modules_experimental.go index 044a27093e..953dccea95 100644 --- a/pkg/pyroscope/modules_experimental.go +++ b/pkg/pyroscope/modules_experimental.go @@ -82,7 +82,7 @@ func (f *Pyroscope) initQueryFrontendV1() (services.Service, error) { } f.API.RegisterFrontendForQuerierHandler(f.frontend) f.API.RegisterQuerierServiceHandler(spanlogger.NewLogSpanParametersWrapper(f.frontend, queryFrontendLogger)) - f.API.RegisterPyroscopeHandlers(spanlogger.NewLogSpanParametersWrapper(f.frontend, queryFrontendLogger)) + f.API.RegisterPyroscopeHandlers(spanlogger.NewLogSpanParametersWrapper(f.frontend, queryFrontendLogger), f.Cfg.Querier.MinStepDuration) f.API.RegisterVCSServiceHandler(f.frontend) return f.frontend, nil } @@ -104,7 +104,7 @@ func (f *Pyroscope) initQueryFrontendV2() (services.Service, error) { ) f.API.RegisterQuerierServiceHandler(spanlogger.NewLogSpanParametersWrapper(queryFrontend, queryFrontendLogger)) - f.API.RegisterPyroscopeHandlers(spanlogger.NewLogSpanParametersWrapper(queryFrontend, queryFrontendLogger)) + f.API.RegisterPyroscopeHandlers(spanlogger.NewLogSpanParametersWrapper(queryFrontend, queryFrontendLogger), f.Cfg.Querier.MinStepDuration) f.API.RegisterVCSServiceHandler(vcsService) // New query frontend does not have any state. @@ -148,7 +148,7 @@ func (f *Pyroscope) initQueryFrontendV12() (services.Service, error) { f.API.RegisterFrontendForQuerierHandler(f.frontend) f.API.RegisterQuerierServiceHandler(spanlogger.NewLogSpanParametersWrapper(handler, queryFrontendLogger)) - f.API.RegisterPyroscopeHandlers(spanlogger.NewLogSpanParametersWrapper(handler, queryFrontendLogger)) + f.API.RegisterPyroscopeHandlers(spanlogger.NewLogSpanParametersWrapper(handler, queryFrontendLogger), f.Cfg.Querier.MinStepDuration) f.API.RegisterVCSServiceHandler(vcsService) return f.frontend, nil diff --git a/pkg/querier/http.go b/pkg/querier/http.go index 53e54a2855..fb8370d898 100644 --- a/pkg/querier/http.go +++ b/pkg/querier/http.go @@ -8,6 +8,7 @@ import ( "net/http" "strconv" "strings" + "time" "connectrpc.com/connect" "github.com/gogo/status" @@ -31,12 +32,16 @@ import ( httputil "github.com/grafana/pyroscope/pkg/util/http" ) -func NewHTTPHandlers(client querierv1connect.QuerierServiceClient) *QueryHandlers { - return &QueryHandlers{client} +func NewHTTPHandlers(client querierv1connect.QuerierServiceClient, minStepDuration time.Duration) *QueryHandlers { + return &QueryHandlers{ + client: client, + minStepDuration: minStepDuration, + } } type QueryHandlers struct { - client querierv1connect.QuerierServiceClient + client querierv1connect.QuerierServiceClient + minStepDuration time.Duration } // LabelValues only returns the label values for the given label name. @@ -186,7 +191,7 @@ func (q *QueryHandlers) Render(w http.ResponseWriter, req *http.Request) { return err }) - timelineStep := timeline.CalcPointInterval(selectParams.Start, selectParams.End) + timelineStep := timeline.CalcPointIntervalWithMinInterval(selectParams.Start, selectParams.End, q.minStepDuration) var resSeries *connect.Response[querierv1.SelectSeriesResponse] g.Go(func() error { var err error diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index c62c88adbb..60995d7c0f 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -45,14 +45,16 @@ import ( ) type Config struct { - PoolConfig clientpool.PoolConfig `yaml:"pool_config,omitempty"` - QueryStoreAfter time.Duration `yaml:"query_store_after" category:"advanced"` + PoolConfig clientpool.PoolConfig `yaml:"pool_config,omitempty"` + QueryStoreAfter time.Duration `yaml:"query_store_after" category:"advanced"` + MinStepDuration time.Duration `yaml:"min_step_duration" category:"advanced"` } // RegisterFlags registers distributor-related flags. func (cfg *Config) RegisterFlags(fs *flag.FlagSet) { cfg.PoolConfig.RegisterFlagsWithPrefix("querier", fs) fs.DurationVar(&cfg.QueryStoreAfter, "querier.query-store-after", 4*time.Hour, "The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.") + fs.DurationVar(&cfg.MinStepDuration, "querier.min-step-duration", 15*time.Second, "Minimum step duration for time series queries. This is the minimum resolution/interval displayed in the UI timeline. Lower values allow for finer-grained profiling resolution when using fast collection intervals (e.g., eBPF with collect_interval < 15s).") } type Limits interface { diff --git a/pkg/querier/timeline/calculator.go b/pkg/querier/timeline/calculator.go index 5e7b32536f..df4237dbab 100644 --- a/pkg/querier/timeline/calculator.go +++ b/pkg/querier/timeline/calculator.go @@ -12,18 +12,29 @@ var ( ) // CalcPointInterval calculates the appropriate interval between each point (aka step) +// using the default minimum interval of 15 seconds. // Note that its main usage is with SelectSeries, therefore its // * inputs are in ms // * output is in seconds func CalcPointInterval(fromMs int64, untilMs int64) float64 { + return CalcPointIntervalWithMinInterval(fromMs, untilMs, DefaultMinInterval) +} + +// CalcPointIntervalWithMinInterval calculates the appropriate interval between each point (aka step) +// with a custom minimum interval. This allows for finer-grained resolution when using fast +// collection intervals (e.g., eBPF with collect_interval < 15s). +// Note that its main usage is with SelectSeries, therefore its +// * inputs are in ms +// * output is in seconds +func CalcPointIntervalWithMinInterval(fromMs int64, untilMs int64, minInterval time.Duration) float64 { resolution := DefaultRes fromNano := fromMs * 1000000 untilNano := untilMs * 1000000 calculatedIntervalNano := time.Duration((untilNano - fromNano) / resolution) - if calculatedIntervalNano < DefaultMinInterval { - return DefaultMinInterval.Seconds() + if calculatedIntervalNano < minInterval { + return minInterval.Seconds() } return roundInterval(calculatedIntervalNano).Seconds() diff --git a/pkg/querier/timeline/calculator_test.go b/pkg/querier/timeline/calculator_test.go index 509f114d87..74d8d90998 100644 --- a/pkg/querier/timeline/calculator_test.go +++ b/pkg/querier/timeline/calculator_test.go @@ -38,3 +38,39 @@ func Test_CalcPointInterval(t *testing.T) { } } + +func Test_CalcPointIntervalWithMinInterval(t *testing.T) { + TestDate := time.Date(2023, time.April, 18, 1, 2, 3, 4, time.UTC) + + testCases := []struct { + name string + start time.Time + end time.Time + minInterval time.Duration + want int64 + }{ + // eBPF use case: 5 second minimum interval + {name: "5s min interval - 1 second", start: TestDate, end: TestDate.Add(1 * time.Second), minInterval: 5 * time.Second, want: 5}, + {name: "5s min interval - 1 hour", start: TestDate, end: TestDate.Add(1 * time.Hour), minInterval: 5 * time.Second, want: 5}, + {name: "5s min interval - 7 days", start: TestDate, end: TestDate.Add(7 * 24 * time.Hour), minInterval: 5 * time.Second, want: 300}, + {name: "5s min interval - 30 days", start: TestDate, end: TestDate.Add(30 * 24 * time.Hour), minInterval: 5 * time.Second, want: 1800}, + + // 1 second minimum interval + {name: "1s min interval - 1 second", start: TestDate, end: TestDate.Add(1 * time.Second), minInterval: 1 * time.Second, want: 1}, + {name: "1s min interval - 10 seconds", start: TestDate, end: TestDate.Add(10 * time.Second), minInterval: 1 * time.Second, want: 1}, + {name: "1s min interval - 1 hour", start: TestDate, end: TestDate.Add(1 * time.Hour), minInterval: 1 * time.Second, want: 2}, + + // Default 15 second minimum interval (should match Test_CalcPointInterval) + {name: "15s min interval - 1 second", start: TestDate, end: TestDate.Add(1 * time.Second), minInterval: 15 * time.Second, want: 15}, + {name: "15s min interval - 1 hour", start: TestDate, end: TestDate.Add(1 * time.Hour), minInterval: 15 * time.Second, want: 15}, + {name: "15s min interval - 7 days", start: TestDate, end: TestDate.Add(7 * 24 * time.Hour), minInterval: 15 * time.Second, want: 300}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := timeline.CalcPointIntervalWithMinInterval(tc.start.UnixMilli(), tc.end.UnixMilli(), tc.minInterval) + + assert.Equal(t, float64(tc.want), got) + }) + } +}