kvserver: delete per action priority inversion metrics

This commit removes per-action priority inversion metrics due to their high
cardinality. We already have logging in place, which should provide sufficient
observability. For now, we care about is priority inversion that leads to
consider rebalance and requeuing the most.

Author: wenyihu6

docs/generated/metrics/metrics.yaml

@@ -13951,118 +13951,6 @@ layers:
       unit: COUNT
       aggregation: AVG
       derivative: NONE
-    - name: queue.replicate.priority_inversion.addnonvoter
-      exported_name: queue_replicate_priority_inversion_addnonvoter
-      description: Number of priority inversions in the replicate queue that resulted in add non-voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.addvoter
-      exported_name: queue_replicate_priority_inversion_addvoter
-      description: Number of priority inversions in the replicate queue that resulted in add voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.considerrebalance
-      exported_name: queue_replicate_priority_inversion_considerrebalance
-      description: Number of priority inversions in the replicate queue that resulted in consider rebalance action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.noop
-      exported_name: queue_replicate_priority_inversion_noop
-      description: Number of priority inversions in the replicate queue that resulted in noop action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.rangeunavailable
-      exported_name: queue_replicate_priority_inversion_rangeunavailable
-      description: Number of priority inversions in the replicate queue that resulted in range unavailable action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.removedeadnonvoter
-      exported_name: queue_replicate_priority_inversion_removedeadnonvoter
-      description: Number of priority inversions in the replicate queue that resulted in remove dead non-voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.removedeadvoter
-      exported_name: queue_replicate_priority_inversion_removedeadvoter
-      description: Number of priority inversions in the replicate queue that resulted in remove dead voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.removedecommissioningnonvoter
-      exported_name: queue_replicate_priority_inversion_removedecommissioningnonvoter
-      description: Number of priority inversions in the replicate queue that resulted in remove decommissioning non-voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.removedecommissioningvoter
-      exported_name: queue_replicate_priority_inversion_removedecommissioningvoter
-      description: Number of priority inversions in the replicate queue that resulted in remove decommissioning voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.removenonvoter
-      exported_name: queue_replicate_priority_inversion_removenonvoter
-      description: Number of priority inversions in the replicate queue that resulted in remove non-voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.removevoter
-      exported_name: queue_replicate_priority_inversion_removevoter
-      description: Number of priority inversions in the replicate queue that resulted in remove voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.replacedeadnonvoter
-      exported_name: queue_replicate_priority_inversion_replacedeadnonvoter
-      description: Number of priority inversions in the replicate queue that resulted in replace dead non-voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.replacedecommissioningnonvoter
-      exported_name: queue_replicate_priority_inversion_replacedecommissioningnonvoter
-      description: Number of priority inversions in the replicate queue that resulted in replace decommissioning non-voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-    - name: queue.replicate.priority_inversion.replacedecommissioningvoter
-      exported_name: queue_replicate_priority_inversion_replacedecommissioningvoter
-      description: Number of priority inversions in the replicate queue that resulted in replace decommissioning voter action during processing
-      y_axis_label: Replicas
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
     - name: queue.replicate.priority_inversion.requeue
       exported_name: queue_replicate_priority_inversion_requeue
       description: Number of priority inversions in the replicate queue that resulted in requeuing of the replicas. A priority inversion occurs when the priority at processing time ends up being lower than at enqueue time. When the priority has changed from a high priority repair action to rebalance, the change is requeued to avoid unfairness.

pkg/kv/kvserver/replicate_queue.go

@@ -338,90 +338,6 @@ var (
 		Measurement: "Replicas",
 		Unit:        metric.Unit_COUNT,
 	}
-	metaReplicateQueuePriorityInversionForAddVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.addvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in add voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForReplaceDecommissioningVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.replacedecommissioningvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in replace decommissioning voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForRemoveDeadVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.removedeadvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in remove dead voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForRemoveDecommissioningVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.removedecommissioningvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in remove decommissioning voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForRemoveVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.removevoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in remove voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForReplaceDeadNonVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.replacedeadnonvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in replace dead non-voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForAddNonVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.addnonvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in add non-voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForReplaceDecommissioningNonVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.replacedecommissioningnonvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in replace decommissioning non-voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForRemoveDeadNonVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.removedeadnonvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in remove dead non-voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForRemoveDecommissioningNonVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.removedecommissioningnonvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in remove decommissioning non-voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForRemoveNonVoterCount = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.removenonvoter",
-		Help:        "Number of priority inversions in the replicate queue that resulted in remove non-voter action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForConsiderRebalance = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.considerrebalance",
-		Help:        "Number of priority inversions in the replicate queue that resulted in consider rebalance action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForRangeUnavailable = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.rangeunavailable",
-		Help:        "Number of priority inversions in the replicate queue that resulted in range unavailable action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
-	metaReplicateQueuePriorityInversionForNoop = metric.Metadata{
-		Name:        "queue.replicate.priority_inversion.noop",
-		Help:        "Number of priority inversions in the replicate queue that resulted in noop action during processing",
-		Measurement: "Replicas",
-		Unit:        metric.Unit_COUNT,
-	}
 )
 
 // quorumError indicates a retryable error condition which sends replicas being
@@ -483,26 +399,9 @@ type ReplicateQueueMetrics struct {
 	// AllocatorConsiderRebalance, and AllocatorFinalizeAtomicReplicationChange
 	// allocator actions.
 
-	// Priority Inversion. Not tracked for
-	// AllocatorFinalizeAtomicReplicationChange, AllocatorRemoveLearner,
-	// AllocatorReplaceDeadVoter since they are the highest priority actions and
-	// cannot be inverted. (17 total actions-3=14)
-	RequeueDueToPriorityInversion                           *metric.Counter
-	PriorityInversionTotal                                  *metric.Counter
-	PriorityInversionForAddVoterCount                       *metric.Counter
-	PriorityInversionForReplaceDecommissioningVoterCount    *metric.Counter
-	PriorityInversionForRemoveDeadVoterCount                *metric.Counter
-	PriorityInversionForRemoveDecommissioningVoterCount     *metric.Counter
-	PriorityInversionForRemoveVoterCount                    *metric.Counter
-	PriorityInversionForReplaceDeadNonVoterCount            *metric.Counter
-	PriorityInversionForAddNonVoterCount                    *metric.Counter
-	PriorityInversionForReplaceDecommissioningNonVoterCount *metric.Counter
-	PriorityInversionForRemoveDeadNonVoterCount             *metric.Counter
-	PriorityInversionForRemoveDecommissioningNonVoterCount  *metric.Counter
-	PriorityInversionForRemoveNonVoterCount                 *metric.Counter
-	PriorityInversionForConsiderRebalance                   *metric.Counter
-	PriorityInversionForRangeUnavailable                    *metric.Counter
-	PriorityInversionForNoop                                *metric.Counter
+	// Priority Inversion.
+	RequeueDueToPriorityInversion *metric.Counter
+	PriorityInversionTotal        *metric.Counter
 }
 
 func makeReplicateQueueMetrics() ReplicateQueueMetrics {
@@ -540,22 +439,8 @@ func makeReplicateQueueMetrics() ReplicateQueueMetrics {
 		RemoveDecommissioningReplicaSuccessCount:  metric.NewCounter(metaReplicateQueueRemoveDecommissioningReplicaSuccessCount),
 		RemoveDecommissioningReplicaErrorCount:    metric.NewCounter(metaReplicateQueueRemoveDecommissioningReplicaErrorCount),
 
-		RequeueDueToPriorityInversion:                           metric.NewCounter(metaReplicateQueueRequeueDueToPriorityInversion),
-		PriorityInversionTotal:                                  metric.NewCounter(metaReplicateQueuePriorityInversionTotal),
-		PriorityInversionForAddVoterCount:                       metric.NewCounter(metaReplicateQueuePriorityInversionForAddVoterCount),
-		PriorityInversionForReplaceDecommissioningVoterCount:    metric.NewCounter(metaReplicateQueuePriorityInversionForReplaceDecommissioningVoterCount),
-		PriorityInversionForRemoveDeadVoterCount:                metric.NewCounter(metaReplicateQueuePriorityInversionForRemoveDeadVoterCount),
-		PriorityInversionForRemoveDecommissioningVoterCount:     metric.NewCounter(metaReplicateQueuePriorityInversionForRemoveDecommissioningVoterCount),
-		PriorityInversionForRemoveVoterCount:                    metric.NewCounter(metaReplicateQueuePriorityInversionForRemoveVoterCount),
-		PriorityInversionForReplaceDeadNonVoterCount:            metric.NewCounter(metaReplicateQueuePriorityInversionForReplaceDeadNonVoterCount),
-		PriorityInversionForAddNonVoterCount:                    metric.NewCounter(metaReplicateQueuePriorityInversionForAddNonVoterCount),
-		PriorityInversionForReplaceDecommissioningNonVoterCount: metric.NewCounter(metaReplicateQueuePriorityInversionForReplaceDecommissioningNonVoterCount),
-		PriorityInversionForRemoveDeadNonVoterCount:             metric.NewCounter(metaReplicateQueuePriorityInversionForRemoveDeadNonVoterCount),
-		PriorityInversionForRemoveDecommissioningNonVoterCount:  metric.NewCounter(metaReplicateQueuePriorityInversionForRemoveDecommissioningNonVoterCount),
-		PriorityInversionForRemoveNonVoterCount:                 metric.NewCounter(metaReplicateQueuePriorityInversionForRemoveNonVoterCount),
-		PriorityInversionForConsiderRebalance:                   metric.NewCounter(metaReplicateQueuePriorityInversionForConsiderRebalance),
-		PriorityInversionForRangeUnavailable:                    metric.NewCounter(metaReplicateQueuePriorityInversionForRangeUnavailable),
-		PriorityInversionForNoop:                                metric.NewCounter(metaReplicateQueuePriorityInversionForNoop),
+		RequeueDueToPriorityInversion: metric.NewCounter(metaReplicateQueueRequeueDueToPriorityInversion),
+		PriorityInversionTotal:        metric.NewCounter(metaReplicateQueuePriorityInversionTotal),
 	}
 }
 
@@ -679,47 +564,6 @@ func (metrics *ReplicateQueueMetrics) trackErrorByAllocatorAction(
 
 }
 
-// trackPriorityInversion tracks the action that the replicate queue ended up
-// processing when the priority at enqueue time was higher than the priority at
-// processing time.
-func (metrics *ReplicateQueueMetrics) trackPriorityInversion(
-	actionAtProcessingTime allocatorimpl.AllocatorAction,
-) {
-	metrics.PriorityInversionTotal.Inc(1)
-	switch actionAtProcessingTime {
-	case allocatorimpl.AllocatorAddVoter:
-		metrics.PriorityInversionForAddVoterCount.Inc(1)
-	case allocatorimpl.AllocatorReplaceDecommissioningVoter:
-		metrics.PriorityInversionForReplaceDecommissioningVoterCount.Inc(1)
-	case allocatorimpl.AllocatorRemoveDeadVoter:
-		metrics.PriorityInversionForRemoveDeadVoterCount.Inc(1)
-	case allocatorimpl.AllocatorRemoveDecommissioningVoter:
-		metrics.PriorityInversionForRemoveDecommissioningVoterCount.Inc(1)
-	case allocatorimpl.AllocatorRemoveVoter:
-		metrics.PriorityInversionForRemoveVoterCount.Inc(1)
-	case allocatorimpl.AllocatorReplaceDeadNonVoter:
-		metrics.PriorityInversionForReplaceDeadNonVoterCount.Inc(1)
-	case allocatorimpl.AllocatorAddNonVoter:
-		metrics.PriorityInversionForAddNonVoterCount.Inc(1)
-	case allocatorimpl.AllocatorReplaceDecommissioningNonVoter:
-		metrics.PriorityInversionForReplaceDecommissioningNonVoterCount.Inc(1)
-	case allocatorimpl.AllocatorRemoveDeadNonVoter:
-		metrics.PriorityInversionForRemoveDeadNonVoterCount.Inc(1)
-	case allocatorimpl.AllocatorRemoveDecommissioningNonVoter:
-		metrics.PriorityInversionForRemoveDecommissioningNonVoterCount.Inc(1)
-	case allocatorimpl.AllocatorRemoveNonVoter:
-		metrics.PriorityInversionForRemoveNonVoterCount.Inc(1)
-	case allocatorimpl.AllocatorConsiderRebalance:
-		metrics.PriorityInversionForConsiderRebalance.Inc(1)
-	case allocatorimpl.AllocatorRangeUnavailable:
-		metrics.PriorityInversionForRangeUnavailable.Inc(1)
-	case allocatorimpl.AllocatorNoop:
-		metrics.PriorityInversionForNoop.Inc(1)
-	default:
-		panic("unhandled default case")
-	}
-}
-
 // trackProcessResult increases the corresponding success/error count metric for
 // processing a particular allocator action through the replicate queue.
 func (metrics *ReplicateQueueMetrics) trackResultByAllocatorAction(
@@ -1135,7 +979,7 @@ func (rq *replicateQueue) processOneChange(
 	// starving other higher priority work.
 	if PriorityInversionRequeue.Get(&rq.store.cfg.Settings.SV) {
 		if inversion, shouldRequeue := allocatorimpl.CheckPriorityInversion(priorityAtEnqueue, change.Action); inversion {
-			rq.metrics.trackPriorityInversion(change.Action)
+			rq.metrics.PriorityInversionTotal.Inc(1)
 			if priorityInversionLogEveryN.ShouldLog() {
 				log.KvDistribution.Infof(ctx,
 					"priority inversion during process: shouldRequeue = %t action=%s, priority=%v, enqueuePriority=%v",

pkg/kv/kvserver/replicate_queue_test.go

@@ -2568,6 +2568,7 @@ func TestReplicateQueueDecommissionScannerDisabled(t *testing.T) {
 func TestPriorityInversionRequeue(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
+	skip.UnderDuress(t)
 
 	ctx := context.Background()
 	settings := cluster.MakeTestingClusterSettings()
@@ -2647,9 +2648,6 @@ func TestPriorityInversionRequeue(t *testing.T) {
 		if c := store.ReplicateQueueMetrics().PriorityInversionTotal.Count(); c == 0 {
 			return errors.New("expected non-zero priority inversion total count but got 0")
 		}
-		if c := store.ReplicateQueueMetrics().PriorityInversionForConsiderRebalance.Count(); c == 0 {
-			return errors.New("expected non-zero priority inversion count for consider rebalance but got 0")
-		}
 		if c := store.ReplicateQueueMetrics().RequeueDueToPriorityInversion.Count(); c == 0 {
 			return errors.New("expected to requeue due to priority inversion but got 0")
 		}