Skip to content

Commit 13ab6fd

Browse files
DrewKimballDrewKimball
committed
sql/opt: add telemetry counters for selectivity clamping
This commit adds telemetry counters for the new histogram selectivity clamping behavior, as well as log messages to indicate when the new behavior applies in a query's trace. Epic: None Release note: None
1 parent 8beb5b1 commit 13ab6fd

File tree

5 files changed

+167
-0
lines changed

5 files changed

+167
-0
lines changed

‎pkg/sql/opt/memo/memo.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,10 @@ type Memo struct {
238238
// erring with partially normalized expressions.
239239
disableCheckExpr bool
240240

241+
// optimizationStats tracks decisions made during optimization, for example,
242+
// to clamp selectivity estimates to a lower bound.
243+
optimizationStats OptimizationStats
244+
241245
// WARNING: if you add more members, add initialization code in Init (if
242246
// reusing allocated data structures is desired).
243247
}
@@ -698,6 +702,25 @@ func (m *Memo) FormatExpr(expr opt.Expr) string {
698702
return f.Buffer.String()
699703
}
700704

705+
// OptimizationStats surfaces information about choices made during optimization
706+
// of a query for top-level observability (e.g. metrics, EXPLAIN output).
707+
type OptimizationStats struct {
708+
// ClampedHistogramSelectivity is true if the selectivity estimate based on a
709+
// histogram was prevented from dropping too low. See also the session var
710+
// "optimizer_clamp_low_histogram_selectivity".
711+
ClampedHistogramSelectivity bool
712+
// ClampedInequalitySelectivity is true if the selectivity estimate for an
713+
// inequality unbounded on one or both sides was prevented from dropping too
714+
// low. See also the session var "optimizer_clamp_inequality_selectivity".
715+
ClampedInequalitySelectivity bool
716+
}
717+
718+
// GetOptimizationStats returns the OptimizationStats collected during a
719+
// previous optimization pass.
720+
func (m *Memo) GetOptimizationStats() *OptimizationStats {
721+
return &m.optimizationStats
722+
}
723+
701724
// ValuesContainer lets ValuesExpr and LiteralValuesExpr share code.
702725
type ValuesContainer interface {
703726
RelExpr

‎pkg/sql/opt/memo/statistics_builder.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4637,6 +4637,9 @@ func (sb *statisticsBuilder) clampSelForHistogram(
46374637
resClamp = props.MinSelectivity(resClamp,
46384638
props.MakeSelectivityFromFraction(oldHist.Resolution(), s.RowCount),
46394639
)
4640+
if resClamp.AsFloat() > clampedSel.AsFloat() {
4641+
sb.mem.optimizationStats.ClampedHistogramSelectivity = true
4642+
}
46404643
clampedSel = props.MaxSelectivity(clampedSel, resClamp)
46414644
}
46424645

@@ -4647,6 +4650,9 @@ func (sb *statisticsBuilder) clampSelForHistogram(
46474650
// scan at least 1/10000th of the table. This accounts for the possibility
46484651
// that the histogram missed extreme values due to sampling or staleness.
46494652
inequalityClamp := props.MakeSelectivity(histogramUnboundedInequalityMinSelectivity)
4653+
if inequalityClamp.AsFloat() > clampedSel.AsFloat() {
4654+
sb.mem.optimizationStats.ClampedInequalitySelectivity = true
4655+
}
46504656
clampedSel = props.MaxSelectivity(clampedSel, inequalityClamp)
46514657
}
46524658
return clampedSel

‎pkg/sql/plan_opt.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,18 @@ func (p *planner) makeOptimizerPlan(ctx context.Context) error {
263263
return err
264264
}
265265

266+
// Log and increment telemetry counters for interesting decisions made during
267+
// optimization.
268+
optStats := execMemo.GetOptimizationStats()
269+
if optStats.ClampedHistogramSelectivity {
270+
log.VEventf(ctx, 2, "clamped histogram selectivity during optimization")
271+
telemetry.Inc(sqltelemetry.PlanClampedHistogramSelectivityCounter)
272+
}
273+
if optStats.ClampedInequalitySelectivity {
274+
log.VEventf(ctx, 2, "clamped inequality selectivity during optimization")
275+
telemetry.Inc(sqltelemetry.PlanClampedInequalitySelectivityCounter)
276+
}
277+
266278
// Build the plan tree.
267279
const disableTelemetryAndPlanGists = false
268280
return p.runExecBuild(ctx, execMemo, disableTelemetryAndPlanGists)

‎pkg/sql/sqltelemetry/planning.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,16 @@ var PlanTypeAutoCustomCounter = telemetry.GetCounterOnce("sql.plan.type.auto-cus
220220
// used when plan_cache_mode=auto.
221221
var PlanTypeAutoGenericCounter = telemetry.GetCounterOnce("sql.plan.type.auto-generic")
222222

223+
// PlanClampedHistogramSelectivityCounter is to be incremented whenever a plan
224+
// is used that clamped the selectivity estimate derived from a histogram to a
225+
// minimum value.
226+
var PlanClampedHistogramSelectivityCounter = telemetry.GetCounterOnce("sql.plan.clamped-histogram-selectivity")
227+
228+
// PlanClampedInequalitySelectivityCounter is to be incremented whenever a plan
229+
// is used that clamped the selectivity estimate derived from an inequality over
230+
// a histogram to a minimum value.
231+
var PlanClampedInequalitySelectivityCounter = telemetry.GetCounterOnce("sql.plan.clamped-inequality-selectivity")
232+
223233
// We can't parameterize these telemetry counters, so just make a bunch of
224234
// buckets for setting the join reorder limit since the range of reasonable
225235
// values for the join reorder limit is quite small.

‎pkg/sql/testdata/telemetry/planning‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,3 +321,119 @@ sql.plan.opt.order-by-nulls-non-standard
321321
feature-usage
322322
SELECT a FROM x ORDER BY b DESC NULLS LAST
323323
----
324+
325+
# Test for clamped histogram selectivity.
326+
327+
feature-list
328+
sql.plan.clamped-histogram-selectivity
329+
sql.plan.clamped-inequality-selectivity
330+
----
331+
332+
exec
333+
SET optimizer_clamp_low_histogram_selectivity = true;
334+
----
335+
336+
exec
337+
SET optimizer_clamp_inequality_selectivity = true;
338+
----
339+
340+
exec
341+
CREATE TYPE status_enum AS ENUM ('ACKNOWLEDGED', 'DROPPED', 'PENDING', 'SUCCEEDED');
342+
----
343+
344+
exec
345+
CREATE TABLE t_large (
346+
k INT PRIMARY KEY,
347+
i INT,
348+
s STRING,
349+
e status_enum,
350+
INDEX (i),
351+
INDEX (s),
352+
INDEX (i) WHERE e = 'PENDING'
353+
)
354+
----
355+
356+
exec
357+
ALTER TABLE t_large INJECT STATISTICS '[
358+
{
359+
"columns": ["k"],
360+
"created_at": "2018-01-01 1:00:00.00000+00:00",
361+
"row_count": 1000000000,
362+
"distinct_count": 1000000000,
363+
"null_count": 0,
364+
"avg_size": 2,
365+
"histo_col_type": "int",
366+
"histo_buckets": [
367+
{"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "0"},
368+
{"num_eq": 1, "num_range": 99000000, "distinct_range": 99000000, "upper_bound": "100000000"},
369+
{"num_eq": 1, "num_range": 199000000, "distinct_range": 199000000, "upper_bound": "300000000"},
370+
{"num_eq": 1, "num_range": 299000000, "distinct_range": 299000000, "upper_bound": "600000000"},
371+
{"num_eq": 1, "num_range": 399000000, "distinct_range": 399000000, "upper_bound": "1000000000"}
372+
]
373+
},
374+
{
375+
"columns": ["i"],
376+
"created_at": "2018-01-01 1:00:00.00000+00:00",
377+
"row_count": 1000000000,
378+
"distinct_count": 44000000,
379+
"null_count": 0,
380+
"avg_size": 2,
381+
"histo_col_type": "int",
382+
"histo_buckets": [
383+
{"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "0"},
384+
{"num_eq": 10000000, "num_range": 90000000, "distinct_range": 10000000, "upper_bound": "100000000"},
385+
{"num_eq": 10000000, "num_range": 190000000, "distinct_range": 10000000, "upper_bound": "200000000"},
386+
{"num_eq": 20000000, "num_range": 280000000, "distinct_range": 10000000, "upper_bound": "300000000"},
387+
{"num_eq": 30000000, "num_range": 370000000, "distinct_range": 10000000, "upper_bound": "400000000"}
388+
]
389+
},
390+
{
391+
"columns": ["s"],
392+
"created_at": "2018-01-01 1:00:00.00000+00:00",
393+
"row_count": 1000000000,
394+
"distinct_count": 400000000,
395+
"null_count": 0,
396+
"avg_size": 3,
397+
"histo_col_type": "string",
398+
"histo_buckets": [
399+
{"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "apple"},
400+
{"num_eq": 100000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "banana"},
401+
{"num_eq": 100000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "cherry"},
402+
{"num_eq": 200000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "mango"},
403+
{"num_eq": 200000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "pineapple"}
404+
]
405+
},
406+
{
407+
"columns": ["e"],
408+
"created_at": "2018-01-01 1:00:00.00000+00:00",
409+
"row_count": 1000000000,
410+
"distinct_count": 4,
411+
"null_count": 0,
412+
"avg_size": 1,
413+
"histo_col_type": "status_enum",
414+
"histo_buckets": [
415+
{"num_eq": 1000000, "num_range": 0, "distinct_range": 0, "upper_bound": "ACKNOWLEDGED"},
416+
{"num_eq": 99000000, "num_range": 0, "distinct_range": 0, "upper_bound": "DROPPED"},
417+
{"num_eq": 900000000, "num_range": 0, "distinct_range": 0, "upper_bound": "SUCCEEDED"}
418+
]
419+
}
420+
]'
421+
----
422+
423+
feature-usage
424+
SELECT * FROM t_large WHERE e = 'PENDING' AND k IN (110000000, 120000000, 130000000, 140000000)
425+
----
426+
sql.plan.clamped-histogram-selectivity
427+
428+
feature-usage
429+
SELECT * FROM t_large WHERE k IN (100000000, 110000000, 120000000, 130000000) AND i > 500000000
430+
----
431+
sql.plan.clamped-histogram-selectivity
432+
sql.plan.clamped-inequality-selectivity
433+
434+
# Cases where the selectivity clamps do not apply.
435+
feature-usage
436+
SELECT * FROM t_large WHERE e = 'SUCCEEDED';
437+
SELECT * FROM t_large;
438+
SELECT 1;
439+
----

0 commit comments

Comments
 (0)