Merge pull request #20905 from k8s-infra-cherrypick-robot/cherry-pick-20887-to-release-3.6

[release-3.6] fix duplicate metrics collector registration attempted

Author: ahrtr

server/etcdserver/api/v3rpc/grpc.go

@@ -17,6 +17,7 @@ package v3rpc
 import (
 	"crypto/tls"
 	"math"
+	"sync"
 
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
 	"github.com/prometheus/client_golang/prometheus"
@@ -35,22 +36,19 @@ const (
 	maxSendBytes = math.MaxInt32
 )
 
+var (
+	metricsServerLock   sync.Mutex
+	metricsServerCached *grpc_prometheus.ServerMetrics
+)
+
 func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnaryServerInterceptor, gopts ...grpc.ServerOption) *grpc.Server {
 	var opts []grpc.ServerOption
 	opts = append(opts, grpc.CustomCodec(&codec{}))
 	if tls != nil {
 		opts = append(opts, grpc.Creds(credentials.NewTransportCredential(tls)))
 	}
 
-	var mopts []grpc_prometheus.ServerMetricsOption
-	if s.Cfg.Metrics == "extensive" {
-		mopts = append(mopts, grpc_prometheus.WithServerHandlingTimeHistogram())
-	}
-	serverMetrics := grpc_prometheus.NewServerMetrics(mopts...)
-	err := prometheus.Register(serverMetrics)
-	if err != nil {
-		s.Cfg.Logger.Warn("etcdserver: failed to register grpc metrics", zap.Error(err))
-	}
+	serverMetrics := getServerMetrics(s.Cfg.Metrics, s.Cfg.Logger)
 
 	chainUnaryInterceptors := []grpc.UnaryServerInterceptor{
 		newLogUnaryInterceptor(s),
@@ -95,3 +93,22 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer
 
 	return grpcServer
 }
+
+func getServerMetrics(metricType string, lg *zap.Logger) *grpc_prometheus.ServerMetrics {
+	metricsServerLock.Lock()
+	defer metricsServerLock.Unlock()
+
+	if metricsServerCached == nil {
+		var mopts []grpc_prometheus.ServerMetricsOption
+		if metricType == "extensive" {
+			mopts = append(mopts, grpc_prometheus.WithServerHandlingTimeHistogram())
+		}
+		metricsServerCached = grpc_prometheus.NewServerMetrics(mopts...)
+		err := prometheus.Register(metricsServerCached)
+		if err != nil {
+			lg.Warn("etcdserver: failed to register grpc metrics", zap.Error(err))
+		}
+	}
+
+	return metricsServerCached
+}