feat: Add initial metrics and update dependencies

This commit introduces Prometheus metrics to the scheduler,
starting with a request counter.

Author: JeffLuoo

cmd/epp/main.go

@@ -30,14 +30,17 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/gateway-api-inference-extension/cmd/epp/runner"
 
+	"github.com/llm-d/llm-d-inference-scheduler/pkg/metrics"
 	"github.com/llm-d/llm-d-inference-scheduler/pkg/plugins"
 )
 
 func main() {
 	// Register llm-d-inference-scheduler plugins
 	plugins.RegisterAllPlugins()
 
-	if err := runner.NewRunner().Run(ctrl.SetupSignalHandler()); err != nil {
+	if err := runner.NewRunner().
+		WithCustomCollectors(metrics.GetCollectors()...).
+		Run(ctrl.SetupSignalHandler()); err != nil {
 		os.Exit(1)
 	}
 }

docs/metrics.md

@@ -0,0 +1,25 @@
+# Metrics
+
+The `llm-d-inference-scheduler` exposes the following Prometheus metrics to monitor its behavior and performance, particularly concerning Prefill/Decode Disaggregation.
+
+All metrics are in the `llm_d_inference_scheduler` subsystem.
+
+## Scrape and see the metric
+
+Metrics defined in the scheduler plugin are extention of Inference Gateway metrics. For more details of seeing metrics, see the [Instruction](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/site-src/guides/metrics-and-observability.md).
+
+## Metric Details
+
+### `pd_decision_total`
+
+*   **Type:** Counter
+*   **Labels:**
+    *   `decision_type`: string ("decode-only" or "prefill-decode")
+*   **Release Stage:** ALPHA
+*   **Description:** Counts the number of requests processed, broken down by the Prefill/Decode disaggregation decision.
+    *   `prefill-decode`: The request was split into separate Prefill and Decode stages.
+    *   `decode-only`: The request used the Decode-only path.
+*   **Usage:** Provides a high-level view of how many requests are utilizing the disaggregated path versus the unified path.
+*   **Actionability:**
+    *   Monitor the ratio of "prefill-decode" to "decode-only" to understand the P/D engagement rate.
+    *   Sudden changes in this ratio might indicate configuration issues, changes in workload patterns, or problems with the decision logic.

go.mod

@@ -14,13 +14,15 @@ require (
 	github.com/onsi/ginkgo/v2 v2.27.2
 	github.com/onsi/gomega v1.38.2
 	github.com/openai/openai-go v1.12.0
+	github.com/prometheus/client_golang v1.23.2
 	github.com/stretchr/testify v1.11.1
 	golang.org/x/sync v0.18.0
 	google.golang.org/grpc v1.76.0
 	k8s.io/api v0.34.2
 	k8s.io/apiextensions-apiserver v0.34.2
 	k8s.io/apimachinery v0.34.2
 	k8s.io/client-go v0.34.2
+	k8s.io/component-base v0.34.2
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d
 	sigs.k8s.io/controller-runtime v0.22.4
@@ -67,6 +69,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -76,7 +79,6 @@ require (
 	github.com/pebbe/zmq4 v1.4.0 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.2 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect
@@ -124,7 +126,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.34.2 // indirect
-	k8s.io/component-base v0.34.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect

pkg/metrics/metrics.go

@@ -0,0 +1,38 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	compbasemetrics "k8s.io/component-base/metrics"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/metrics"
+)
+
+const (
+	SchedulerSubsystem = "llm_d_inference_scheduler"
+
+	// Decision types for pd_decision_total metric
+	DecisionTypeDecodeOnly    = "decode-only"
+	DecisionTypePrefillDecode = "prefill-decode"
+)
+
+var (
+	SchedulerPDDecisionCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: SchedulerSubsystem,
+			Name:      "pd_decision_total",
+			Help:      metrics.HelpMsgWithStability("Total number of P/D disaggregation decisions made", compbasemetrics.ALPHA),
+		},
+		[]string{"decision_type"}, // "decode-only" or "prefill-decode"
+	)
+)
+
+// GetCollectors returns all custom collectors for the llm-d-inference-scheduler.
+func GetCollectors() []prometheus.Collector {
+	return []prometheus.Collector{
+		SchedulerPDDecisionCount,
+	}
+}
+
+// RecordPDDecisionCounter records the type of P/D disaggregation decision made.
+func RecordPDDecisionCounter(decisionType string) {
+	SchedulerPDDecisionCount.WithLabelValues(decisionType).Inc()
+}

pkg/metrics/metrics_test.go

@@ -0,0 +1,22 @@
+package metrics
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestSchedulerPDDecisionCount(t *testing.T) {
+	RecordPDDecisionCounter(DecisionTypePrefillDecode)
+	RecordPDDecisionCounter(DecisionTypeDecodeOnly)
+	RecordPDDecisionCounter(DecisionTypePrefillDecode)
+	if err := testutil.CollectAndCompare(SchedulerPDDecisionCount, strings.NewReader(`
+		# HELP llm_d_inference_scheduler_pd_decision_total [ALPHA] Total number of P/D disaggregation decisions made
+		# TYPE llm_d_inference_scheduler_pd_decision_total counter
+		llm_d_inference_scheduler_pd_decision_total{decision_type="decode-only"} 1
+		llm_d_inference_scheduler_pd_decision_total{decision_type="prefill-decode"} 2
+	`), "decision_type"); err != nil {
+		t.Errorf("RecordPDDecisionCounter() failed: %v", err)
+	}
+}

pkg/plugins/profile/pd_profile_handler.go

@@ -9,6 +9,8 @@ import (
 	"net"
 	"strconv"
 
+	"github.com/llm-d/llm-d-inference-scheduler/pkg/metrics"
+
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
@@ -156,10 +158,12 @@ func (h *PdProfileHandler) Pick(ctx context.Context, cycleState *types.CycleStat
 
 		if (1.0-hitPercentagePrefix)*float64(len(userInput)) < float64(h.pdThreshold) {
 			log.FromContext(ctx).Info("Non-cached suffix is smaller than threshold, using decode profile only", "hitPercentage", hitPercentagePrefix)
+			metrics.RecordPDDecisionCounter(metrics.DecisionTypeDecodeOnly)
 			return map[string]*framework.SchedulerProfile{} // do not run prefill
 		}
 	}
 
+	metrics.RecordPDDecisionCounter(metrics.DecisionTypePrefillDecode)
 	// run the prefill profile
 	return map[string]*framework.SchedulerProfile{
 		h.prefillProfile: profiles[h.prefillProfile],