Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions cmd/pd-sidecar/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ import (
"strconv"
"strings"

"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"

"github.com/llm-d/llm-d-inference-scheduler/pkg/sidecar/proxy"
"github.com/llm-d/llm-d-inference-scheduler/pkg/sidecar/version"
Expand Down Expand Up @@ -58,14 +59,15 @@ func main() {
inferencePoolName := flag.String("inference-pool-name", os.Getenv("INFERENCE_POOL_NAME"), "the specific InferencePool name to watch (defaults to INFERENCE_POOL_NAME env var)")
enablePrefillerSampling := flag.Bool("enable-prefiller-sampling", func() bool { b, _ := strconv.ParseBool(os.Getenv("ENABLE_PREFILLER_SAMPLING")); return b }(), "if true, the target prefill instance will be selected randomly from among the provided prefill host values")

klog.InitFlags(nil)
opts := zap.Options{}
opts.BindFlags(flag.CommandLine) // optional to allow zap logging control via CLI
flag.Parse()

// make sure to flush logs before exiting
defer klog.Flush()
logger := zap.New(zap.UseFlagOptions(&opts))
log.SetLogger(logger)

ctx := ctrl.SetupSignalHandler()
logger := klog.FromContext(ctx)
log.IntoContext(ctx, logger)

logger.Info("Proxy starting", "Built on", version.BuildRef, "From Git SHA", version.CommitSHA)

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ require (
k8s.io/apimachinery v0.34.2
k8s.io/client-go v0.34.2
k8s.io/component-base v0.34.2
k8s.io/klog/v2 v2.130.1
k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d
sigs.k8s.io/controller-runtime v0.22.4
sigs.k8s.io/gateway-api v1.4.0
Expand Down Expand Up @@ -126,6 +125,7 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiserver v0.34.2 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
Expand Down
4 changes: 2 additions & 2 deletions pkg/sidecar/proxy/allowlist.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ import (
"k8s.io/client-go/dynamic"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
"k8s.io/utils/set"
"sigs.k8s.io/controller-runtime/pkg/log"
)

const (
Expand Down Expand Up @@ -105,7 +105,7 @@ func (av *AllowlistValidator) Start(ctx context.Context) error {
return nil
}

av.logger = klog.FromContext(ctx).WithName("allowlist-validator")
av.logger = log.FromContext(ctx).WithName("allowlist-validator")
av.logger.Info("starting SSRF protection allowlist validator", "namespace", av.namespace, "poolName", av.poolName)

gvr := schema.GroupVersionResource{
Expand Down
3 changes: 1 addition & 2 deletions pkg/sidecar/proxy/connector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"github.com/llm-d/llm-d-inference-scheduler/test/sidecar/mock"
. "github.com/onsi/ginkgo/v2" // nolint:revive
. "github.com/onsi/gomega" // nolint:revive
"k8s.io/klog/v2/ktesting"
)

type sidecarTestInfo struct {
Expand Down Expand Up @@ -180,7 +179,7 @@ var _ = Describe("Common Connector tests", func() {
func sidecarConnectionTestSetup(connector string) *sidecarTestInfo {
testInfo := sidecarTestInfo{}

_, testInfo.ctx = ktesting.NewTestContext(GinkgoT())
testInfo.ctx = newTestContext()
testInfo.ctx, testInfo.cancelFn = context.WithCancel(testInfo.ctx)
testInfo.stoppedCh = make(chan struct{})

Expand Down
4 changes: 2 additions & 2 deletions pkg/sidecar/proxy/data_parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

"github.com/llm-d/llm-d-inference-scheduler/pkg/common"
"golang.org/x/sync/errgroup"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/log"
)

// dataParallelHandler checks if Data Parallel handling is needed.
Expand Down Expand Up @@ -71,7 +71,7 @@ func (s *Server) startDataParallel(ctx context.Context, cert *tls.Certificate, g
}

clone := s.Clone()
clone.logger = klog.FromContext(ctx).WithName("proxy server on port " + rankPort)
clone.logger = log.FromContext(ctx).WithName("proxy server on port " + rankPort)
clone.port = rankPort
clone.decoderURL = decoderURL
clone.forwardDataParallel = false
Expand Down
3 changes: 1 addition & 2 deletions pkg/sidecar/proxy/data_parallel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
. "github.com/onsi/ginkgo/v2" // nolint:revive
. "github.com/onsi/gomega" // nolint:revive
"golang.org/x/sync/errgroup"
"k8s.io/klog/v2/ktesting"

"github.com/llm-d/llm-d-inference-scheduler/pkg/common"
sidecarmock "github.com/llm-d/llm-d-inference-scheduler/test/sidecar/mock"
Expand All @@ -24,7 +23,7 @@ const (
var _ = Describe("Data Parallel support", func() {
When("configured with --data-parallel-size > 1", func() {
It("should create an extra proxy", func() {
_, ctx := ktesting.NewTestContext(GinkgoT())
ctx := newTestContext()
ctx, cancel := context.WithCancel(ctx)
grp, ctx := errgroup.WithContext(ctx)

Expand Down
4 changes: 2 additions & 2 deletions pkg/sidecar/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
"github.com/go-logr/logr"
lru "github.com/hashicorp/golang-lru/v2"
"golang.org/x/sync/errgroup"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/log"
)

const (
Expand Down Expand Up @@ -141,7 +141,7 @@ func NewProxy(port string, decodeURL *url.URL, config Config) *Server {

// Start the HTTP reverse proxy.
func (s *Server) Start(ctx context.Context, cert *tls.Certificate, allowlistValidator *AllowlistValidator) error {
s.logger = klog.FromContext(ctx).WithName("proxy server on port " + s.port)
s.logger = log.FromContext(ctx).WithName("proxy server on port " + s.port)

s.allowlistValidator = allowlistValidator

Expand Down
21 changes: 17 additions & 4 deletions pkg/sidecar/proxy/proxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,33 @@ import (
"strings"
"time"

"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"

"github.com/llm-d/llm-d-inference-scheduler/pkg/common"
"github.com/llm-d/llm-d-inference-scheduler/test/sidecar/mock"
. "github.com/onsi/ginkgo/v2" // nolint:revive
. "github.com/onsi/gomega" // nolint:revive
"k8s.io/klog/v2/ktesting"
)

func newTestContext() context.Context {
logger := zap.New(
zap.WriteTo(GinkgoWriter),
zap.UseDevMode(true),
)
log.SetLogger(logger)
ctx := context.Background()
log.IntoContext(ctx, logger) // not strictly needed since we called SetLogger to set default
return ctx
}

var _ = Describe("Reverse Proxy", func() {
When("x-prefiller-url is not present", func() {
DescribeTable("should forward requests to decode server",

func(path string, secureProxy bool) {
_, ctx := ktesting.NewTestContext(GinkgoT())

ctx := newTestContext()
var cert *tls.Certificate
if secureProxy {
tempCert, err := CreateSelfSignedTLSCertificate()
Expand Down Expand Up @@ -159,7 +172,7 @@ var _ = Describe("Reverse Proxy", func() {
})

It("should successfully send request to 1. prefill 2. decode with the right fields (backward compatible behavior)", func() {
_, ctx := ktesting.NewTestContext(GinkgoT())
ctx := newTestContext()
ctx, cancelFn := context.WithCancel(ctx)
stoppedCh := make(chan struct{})

Expand Down Expand Up @@ -233,7 +246,7 @@ var _ = Describe("Reverse Proxy", func() {
})

It("should successfully send request to 1. prefill 2. decode with the right fields", func() {
_, ctx := ktesting.NewTestContext(GinkgoT())
ctx := newTestContext()
ctx, cancelFn := context.WithCancel(ctx)
stoppedCh := make(chan struct{})

Expand Down
4 changes: 4 additions & 0 deletions test/utils/network.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
// Package utils contains utilities for testing
//
//revive:disable:var-naming
package utils

//revive:enable:var-naming

import (
"net"
)
Expand Down