Merge pull request #156399 from DrewKimball/backport25.4-153067

release-25.4: opt: make outside-of-histogram estimates more pessimistic

Author: DrewKimball

pkg/sql/exec_util.go

@@ -4447,6 +4447,14 @@ func (m *sessionDataMutator) SetOptimizerUseImprovedHoistJoinProject(val bool) {
 	m.data.OptimizerUseImprovedHoistJoinProject = val
 }
 
+func (m *sessionDataMutator) SetOptimizerClampLowHistogramSelectivity(val bool) {
+	m.data.OptimizerClampLowHistogramSelectivity = val
+}
+
+func (m *sessionDataMutator) SetOptimizerClampInequalitySelectivity(val bool) {
+	m.data.OptimizerClampInequalitySelectivity = val
+}
+
 // Utility functions related to scrubbing sensitive information on SQL Stats.
 
 // quantizeCounts ensures that the Count field in the

pkg/sql/logictest/testdata/logic_test/information_schema

@@ -4189,6 +4189,8 @@ opt_split_scan_limit                                             2048
 optimizer                                                        on
 optimizer_always_use_histograms                                  on
 optimizer_check_input_min_row_count                              1
+optimizer_clamp_inequality_selectivity                           off
+optimizer_clamp_low_histogram_selectivity                        off
 optimizer_disable_cross_region_cascade_fast_path_for_rbr_tables  on
 optimizer_enable_lock_elision                                    on
 optimizer_hoist_uncorrelated_equality_subqueries                 on

pkg/sql/logictest/testdata/logic_test/pg_catalog

@@ -3091,6 +3091,8 @@ on_update_rehome_row_enabled                                     on
 opt_split_scan_limit                                             2048                NULL      NULL        NULL        string
 optimizer_always_use_histograms                                  on                  NULL      NULL        NULL        string
 optimizer_check_input_min_row_count                              1                   NULL      NULL        NULL        string
+optimizer_clamp_inequality_selectivity                           off                 NULL      NULL        NULL        string
+optimizer_clamp_low_histogram_selectivity                        off                 NULL      NULL        NULL        string
 optimizer_disable_cross_region_cascade_fast_path_for_rbr_tables  on                  NULL      NULL        NULL        string
 optimizer_enable_lock_elision                                    on                  NULL      NULL        NULL        string
 optimizer_hoist_uncorrelated_equality_subqueries                 on                  NULL      NULL        NULL        string
@@ -3336,6 +3338,8 @@ on_update_rehome_row_enabled                                     on
 opt_split_scan_limit                                             2048                NULL  user     NULL      2048                2048
 optimizer_always_use_histograms                                  on                  NULL  user     NULL      on                  on
 optimizer_check_input_min_row_count                              1                   NULL  user     NULL      1                   1
+optimizer_clamp_inequality_selectivity                           off                 NULL  user     NULL      off                 off
+optimizer_clamp_low_histogram_selectivity                        off                 NULL  user     NULL      off                 off
 optimizer_disable_cross_region_cascade_fast_path_for_rbr_tables  on                  NULL  user     NULL      on                  on
 optimizer_enable_lock_elision                                    on                  NULL  user     NULL      on                  on
 optimizer_hoist_uncorrelated_equality_subqueries                 on                  NULL  user     NULL      on                  on
@@ -3572,6 +3576,8 @@ opt_split_scan_limit                                             NULL    NULL
 optimizer                                                        NULL    NULL     NULL     NULL        NULL
 optimizer_always_use_histograms                                  NULL    NULL     NULL     NULL        NULL
 optimizer_check_input_min_row_count                              NULL    NULL     NULL     NULL        NULL
+optimizer_clamp_inequality_selectivity                           NULL    NULL     NULL     NULL        NULL
+optimizer_clamp_low_histogram_selectivity                        NULL    NULL     NULL     NULL        NULL
 optimizer_disable_cross_region_cascade_fast_path_for_rbr_tables  NULL    NULL     NULL     NULL        NULL
 optimizer_enable_lock_elision                                    NULL    NULL     NULL     NULL        NULL
 optimizer_hoist_uncorrelated_equality_subqueries                 NULL    NULL     NULL     NULL        NULL

pkg/sql/logictest/testdata/logic_test/show_source

@@ -155,6 +155,8 @@ on_update_rehome_row_enabled                                     on
 opt_split_scan_limit                                             2048
 optimizer_always_use_histograms                                  on
 optimizer_check_input_min_row_count                              1
+optimizer_clamp_inequality_selectivity                           off
+optimizer_clamp_low_histogram_selectivity                        off
 optimizer_disable_cross_region_cascade_fast_path_for_rbr_tables  on
 optimizer_enable_lock_elision                                    on
 optimizer_hoist_uncorrelated_equality_subqueries                 on

pkg/sql/opt/memo/memo.go

@@ -211,6 +211,8 @@ type Memo struct {
 	useExistsFilterHoistRule                   bool
 	disableSlowCascadeFastPathForRBRTables     bool
 	useImprovedHoistJoinProject                bool
+	clampLowHistogramSelectivity               bool
+	clampInequalitySelectivity                 bool
 
 	// txnIsoLevel is the isolation level under which the plan was created. This
 	// affects the planning of some locking operations, so it must be included in
@@ -236,6 +238,10 @@ type Memo struct {
 	// erring with partially normalized expressions.
 	disableCheckExpr bool
 
+	// optimizationStats tracks decisions made during optimization, for example,
+	// to clamp selectivity estimates to a lower bound.
+	optimizationStats OptimizationStats
+
 	// WARNING: if you add more members, add initialization code in Init (if
 	// reusing allocated data structures is desired).
 }
@@ -318,6 +324,8 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
 		useExistsFilterHoistRule:                   evalCtx.SessionData().OptimizerUseExistsFilterHoistRule,
 		disableSlowCascadeFastPathForRBRTables:     evalCtx.SessionData().OptimizerDisableCrossRegionCascadeFastPathForRBRTables,
 		useImprovedHoistJoinProject:                evalCtx.SessionData().OptimizerUseImprovedHoistJoinProject,
+		clampLowHistogramSelectivity:               evalCtx.SessionData().OptimizerClampLowHistogramSelectivity,
+		clampInequalitySelectivity:                 evalCtx.SessionData().OptimizerClampInequalitySelectivity,
 		txnIsoLevel:                                evalCtx.TxnIsoLevel,
 	}
 	m.metadata.Init()
@@ -493,6 +501,8 @@ func (m *Memo) IsStale(
 		m.useExistsFilterHoistRule != evalCtx.SessionData().OptimizerUseExistsFilterHoistRule ||
 		m.disableSlowCascadeFastPathForRBRTables != evalCtx.SessionData().OptimizerDisableCrossRegionCascadeFastPathForRBRTables ||
 		m.useImprovedHoistJoinProject != evalCtx.SessionData().OptimizerUseImprovedHoistJoinProject ||
+		m.clampLowHistogramSelectivity != evalCtx.SessionData().OptimizerClampLowHistogramSelectivity ||
+		m.clampInequalitySelectivity != evalCtx.SessionData().OptimizerClampInequalitySelectivity ||
 		m.txnIsoLevel != evalCtx.TxnIsoLevel {
 		return true, nil
 	}
@@ -692,6 +702,25 @@ func (m *Memo) FormatExpr(expr opt.Expr) string {
 	return f.Buffer.String()
 }
 
+// OptimizationStats surfaces information about choices made during optimization
+// of a query for top-level observability (e.g. metrics, EXPLAIN output).
+type OptimizationStats struct {
+	// ClampedHistogramSelectivity is true if the selectivity estimate based on a
+	// histogram was prevented from dropping too low. See also the session var
+	// "optimizer_clamp_low_histogram_selectivity".
+	ClampedHistogramSelectivity bool
+	// ClampedInequalitySelectivity is true if the selectivity estimate for an
+	// inequality unbounded on one or both sides was prevented from dropping too
+	// low. See also the session var "optimizer_clamp_inequality_selectivity".
+	ClampedInequalitySelectivity bool
+}
+
+// GetOptimizationStats returns the OptimizationStats collected during a
+// previous optimization pass.
+func (m *Memo) GetOptimizationStats() *OptimizationStats {
+	return &m.optimizationStats
+}
+
 // ValuesContainer lets ValuesExpr and LiteralValuesExpr share code.
 type ValuesContainer interface {
 	RelExpr

pkg/sql/opt/memo/memo_test.go

@@ -595,6 +595,16 @@ func TestMemoIsStale(t *testing.T) {
 	evalCtx.SessionData().OptimizerUseImprovedHoistJoinProject = false
 	notStale()
 
+	evalCtx.SessionData().OptimizerClampLowHistogramSelectivity = true
+	stale()
+	evalCtx.SessionData().OptimizerClampLowHistogramSelectivity = false
+	notStale()
+
+	evalCtx.SessionData().OptimizerClampInequalitySelectivity = true
+	stale()
+	evalCtx.SessionData().OptimizerClampInequalitySelectivity = false
+	notStale()
+
 	// User no longer has access to view.
 	catalog.View(tree.NewTableNameWithSchema("t", catconstants.PublicSchemaName, "abcview")).Revoked = true
 	_, err = o.Memo().IsStale(ctx, &evalCtx, catalog)

pkg/sql/opt/memo/statistics_builder.go

@@ -90,6 +90,29 @@ const (
 	// values from the spans a check constraint is allowed to have in order to build
 	// a histogram from it.
 	maxValuesForFullHistogramFromCheckConstraint = tabledesc.MaxBucketAllowed
+
+	// histogramPessimisticThreshold determines the cutoff point below which the
+	// selectivity estimate of a histogram is overridden with a more pessimistic
+	// estimate. This is to avoid over-fitting to a stale or inaccurate histogram.
+	//
+	// The value (1 in 10,000) was chosen because we choose sample sizes according
+	// to table size such that we expect to *nearly* always sample all values with
+	// multiplicity >= row_count/10000. Cardinality estimates below this threshold
+	// are increasingly likely to be inaccurate. See also computeNumberSamples.
+	histogramPessimisticThreshold = 1.0 / 10000.0
+
+	// histogramInequalityMinSelectivity determines the minimum selectivity
+	// estimate that can be derived from an unbounded (above or below) inequality
+	// used to filter a histogram. Similar to histogramPessimisticThreshold, this
+	// is to avoid over-fitting to a stale or inaccurate histogram.
+	//
+	// The value (1 in 10,000) was chosen based on similar logic in Postgres,
+	// which caps the selectivity to (1 / bucket_count*100). Postgres uses 100
+	// histogram buckets by default, so the number comes out to 10,000. We avoid
+	// using the number of histogram buckets directly to avoid arbitrary
+	// variation in the selectivity cap depending on user settings and partial
+	// stat collections.
+	histogramUnboundedInequalityMinSelectivity = 1.0 / 10000.0
 )
 
 // statisticsBuilder is responsible for building the statistics that are
@@ -761,7 +784,10 @@ func (sb *statisticsBuilder) makeTableStatistics(tabID opt.TableID) *props.Stati
 					// is tracked here: https://github.com/cockroachdb/cockroach/issues/50655
 					col := cols.SingleColumn()
 					colStat.Histogram = &props.Histogram{}
-					colStat.Histogram.Init(sb.evalCtx, col, stat.Histogram())
+					// Track the minimum number of rows for which histogram selectivity
+					// estimates are trusted.
+					resolution := histogramPessimisticThreshold * stats.RowCount
+					colStat.Histogram.Init(sb.evalCtx, col, stat.Histogram(), resolution)
 				}
 
 				// Make sure the distinct count is at least 1, for the same reason as
@@ -786,7 +812,18 @@ func (sb *statisticsBuilder) makeTableStatistics(tabID opt.TableID) *props.Stati
 					invCols := opt.MakeColSet(invCol)
 					if invColStat, ok := stats.ColStats.Add(invCols); ok {
 						invColStat.Histogram = &props.Histogram{}
-						invColStat.Histogram.Init(sb.evalCtx, invCol, stat.Histogram())
+						// Track the minimum number of rows for which histogram selectivity
+						// estimates are trusted.
+						//
+						// NOTE: an inverted index can have multiple entries per table row.
+						// However, we still use the number of table rows here because the
+						// max multiplicity of a missed value is proportional to the number
+						// of table rows, not the number of inverted index entries. For
+						// example, the arrays [10, 20, 30] and [20, 40, 60] result in six
+						// inverted index entries, but only a maximum multiplicity of two
+						// for the value "20".
+						resolution := histogramPessimisticThreshold * stats.RowCount
+						invColStat.Histogram.Init(sb.evalCtx, invCol, stat.Histogram(), resolution)
 						// Set inverted entry counts from the histogram. Make sure the
 						// distinct count is at least 1, for the same reason as the row
 						// count above.
@@ -4558,10 +4595,15 @@ func (sb *statisticsBuilder) selectivityFromHistograms(
 		newCount := newHist.ValuesCount()
 		oldCount := oldHist.ValuesCount()
 
-		// Calculate the selectivity of the predicate. Nulls are already included
-		// in the histogram, so we do not need to account for them separately.
+		// Calculate the selectivity of the predicate using the histogram. Nulls
+		// are already included in the histogram, so we do not need to account for
+		// them separately.
 		predicateSelectivity := props.MakeSelectivityFromFraction(newCount, oldCount)
 
+		// Possibly clamp the selectivity to a higher value to avoid overly
+		// optimistic estimates.
+		predicateSelectivity = sb.clampSelForHistogram(inputColStat, colStat, s, predicateSelectivity)
+
 		// The maximum possible selectivity of the entire expression is the minimum
 		// selectivity of all individual predicates.
 		selectivityUpperBound = props.MinSelectivity(selectivityUpperBound, predicateSelectivity)
@@ -4572,6 +4614,50 @@ func (sb *statisticsBuilder) selectivityFromHistograms(
 	return selectivity, selectivityUpperBound
 }
 
+// clampSelForHistogram clamps the selectivity estimate derived from a histogram
+// to a minimum value. This accounts for the possibility that the histogram is
+// missing values due to sampling or staleness. See also
+// histogramPessimisticThreshold.
+func (sb *statisticsBuilder) clampSelForHistogram(
+	oldColStat, newColStat *props.ColumnStatistic, s *props.Statistics, originalSel props.Selectivity,
+) (clampedSel props.Selectivity) {
+	clampedSel = originalSel
+	oldHist, newHist := oldColStat.Histogram, newColStat.Histogram
+	if sb.evalCtx.SessionData().OptimizerClampLowHistogramSelectivity &&
+		newHist.ValuesCount() < oldHist.Resolution() {
+		// NOTE: columns with histograms are skipped when considering distinct
+		// counts in selectivityFromSingleColDistinctCounts, so this doesn't
+		// double count the effect of the predicate.
+		resClamp := props.MakeSelectivityFromFraction(newColStat.DistinctCount, oldColStat.DistinctCount)
+
+		// Cap the selectivity so that the row count estimate is no more than the
+		// pessimistic threshold. This can result in a lower estimate if the
+		// multiplicities of the filtered values really are low compared to the
+		// average multiplicity.
+		resClamp = props.MinSelectivity(resClamp,
+			props.MakeSelectivityFromFraction(oldHist.Resolution(), s.RowCount),
+		)
+		if resClamp.AsFloat() > clampedSel.AsFloat() {
+			sb.mem.optimizationStats.ClampedHistogramSelectivity = true
+		}
+		clampedSel = props.MaxSelectivity(clampedSel, resClamp)
+	}
+
+	tightUpperBound, tightLowerBound := newHist.TightBounds()
+	if sb.evalCtx.SessionData().OptimizerClampInequalitySelectivity &&
+		(!tightUpperBound || !tightLowerBound) {
+		// Similar to Postgres, assume that an open-ended inequality predicate will
+		// scan at least 1/10000th of the table. This accounts for the possibility
+		// that the histogram missed extreme values due to sampling or staleness.
+		inequalityClamp := props.MakeSelectivity(histogramUnboundedInequalityMinSelectivity)
+		if inequalityClamp.AsFloat() > clampedSel.AsFloat() {
+			sb.mem.optimizationStats.ClampedInequalitySelectivity = true
+		}
+		clampedSel = props.MaxSelectivity(clampedSel, inequalityClamp)
+	}
+	return clampedSel
+}
+
 // selectivityFromMaxFrequencies calculates the selectivity of an equality
 // filters by using the maximum frequency of the histograms of the constrained
 // columns. This represents a worst-case selectivity estimate and is used to
@@ -5332,7 +5418,10 @@ func (sb *statisticsBuilder) buildStatsFromCheckConstraints(
 			colStat.NullCount = nullCount
 			if useHistogram {
 				colStat.Histogram = &props.Histogram{}
-				colStat.Histogram.Init(sb.evalCtx, firstColID, histogram)
+				// Track the minimum number of rows for which histogram selectivity
+				// estimates are trusted.
+				resolution := histogramPessimisticThreshold * statistics.RowCount
+				colStat.Histogram.Init(sb.evalCtx, firstColID, histogram, resolution)
 			}
 			sb.finalizeFromRowCountAndDistinctCounts(colStat, statistics)
 			tabMeta.AddCheckConstraintsStats(firstColID, colStat)