Skip to content
This repository was archived by the owner on Dec 3, 2024. It is now read-only.

Commit 2dc3d48

Browse files
tam7ttam7t
committed
Support metric keys with multiple metric types
go-metrics allows the same key to be used for multiple metric types (gauge + histogram for example). Specifically vault exports vault.database.CreateUser as both a histogram and a counter. This expands ExtractLabelsFn hook to take the metric type in as a parameter and will append "counter" or "gauge" to the end of the metric name to avoid collisions.
1 parent fa2b87f commit 2dc3d48

File tree

4 files changed

+78
-30
lines changed

4 files changed

+78
-30
lines changed

‎stackdriver.go‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,10 @@ type taskInfo struct {
108108
// name.
109109
type BucketFn func([]string) []float64
110110

111-
// ExtractLabelsFn converts a given metric name into a new metric name and
112-
// optionally additional labels. Errors will prevent the metric from writing
113-
// to stackdriver.
114-
type ExtractLabelsFn func([]string) ([]string, []metrics.Label, error)
111+
// ExtractLabelsFn converts a given metric name and type into a new metric
112+
// name and optionally additional labels. Errors will prevent the metric from
113+
// writing to stackdriver.
114+
type ExtractLabelsFn func([]string, string) ([]string, []metrics.Label, error)
115115

116116
// DefaultBucketer is the default BucketFn used to determing bucketing values
117117
// for metrics.
@@ -120,9 +120,19 @@ func DefaultBucketer(key []string) []float64 {
120120
}
121121

122122
// DefaultLabelExtractor is the default ExtractLabelsFn and is a direct
123-
// passthrough.
124-
func DefaultLabelExtractor(key []string) ([]string, []metrics.Label, error) {
125-
return key, nil, nil
123+
// passthrough. Counter and Gauge metrics are renamed to include the type in
124+
// their name to avoid duplicate metrics with the same name but different
125+
// types (which is allowed by go-metrics but not by Stackdriver).
126+
func DefaultLabelExtractor(key []string, kind string) ([]string, []metrics.Label, error) {
127+
switch kind {
128+
case "counter":
129+
return append(key, "counter"), nil, nil
130+
case "gauge":
131+
return append(key, "gauge"), nil, nil
132+
case "histogram":
133+
return key, nil, nil
134+
}
135+
return nil, nil, fmt.Errorf("Unknown metric kind: %s", kind)
126136
}
127137

128138
// NewSink creates a Sink to flush metrics to stackdriver every interval. The
@@ -270,7 +280,7 @@ func (s *Sink) report(ctx context.Context) {
270280
ts := []*monitoringpb.TimeSeries{}
271281

272282
for _, v := range rCounters {
273-
name, labels, err := v.name.labelMap(s.extractor)
283+
name, labels, err := v.name.labelMap(s.extractor, "counter")
274284
if err != nil {
275285
log.Printf("Could not extract labels from %s: %v", v.name.hash, err)
276286
continue
@@ -300,7 +310,7 @@ func (s *Sink) report(ctx context.Context) {
300310
}
301311

302312
for _, v := range rGauges {
303-
name, labels, err := v.name.labelMap(s.extractor)
313+
name, labels, err := v.name.labelMap(s.extractor, "gauge")
304314
if err != nil {
305315
log.Printf("Could not extract labels from %s: %v", v.name.hash, err)
306316
continue
@@ -330,7 +340,7 @@ func (s *Sink) report(ctx context.Context) {
330340
}
331341

332342
for _, v := range rHistograms {
333-
name, labels, err := v.name.labelMap(s.extractor)
343+
name, labels, err := v.name.labelMap(s.extractor, "histogram")
334344
if err != nil {
335345
log.Printf("Could not extract labels from %s: %v", v.name.hash, err)
336346
continue
@@ -389,13 +399,17 @@ func (s *Sink) report(ctx context.Context) {
389399
end = len(ts)
390400
}
391401

392-
err := s.client.CreateTimeSeries(ctx, &monitoringpb.CreateTimeSeriesRequest{
402+
req := &monitoringpb.CreateTimeSeriesRequest{
393403
Name: fmt.Sprintf("projects/%s", s.taskInfo.ProjectID),
394404
TimeSeries: ts[i:end],
395-
})
405+
}
406+
err := s.client.CreateTimeSeries(ctx, req)
396407

397408
if err != nil {
398409
log.Printf("Failed to write time series data: %v", err)
410+
for i, a := range req.TimeSeries {
411+
log.Printf("request timeseries[%d]: %v", i, a)
412+
}
399413
}
400414
}
401415
}
@@ -495,9 +509,9 @@ func newSeries(key []string, labels []metrics.Label) *series {
495509
}
496510
}
497511

498-
func (s *series) labelMap(extractor ExtractLabelsFn) (string, map[string]string, error) {
512+
func (s *series) labelMap(extractor ExtractLabelsFn, kind string) (string, map[string]string, error) {
499513
// extract labels from the key
500-
key, extractedLabels, err := extractor(s.key)
514+
key, extractedLabels, err := extractor(s.key, kind)
501515
if err != nil {
502516
return "", nil, err
503517
}
@@ -552,4 +566,4 @@ func (h *histogram) addSample(val float32) {
552566
}
553567

554568
h.counts[len(h.buckets)-1]++
555-
}
569+
}

‎stackdriver_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ func TestSample(t *testing.T) {
281281

282282
func TestExtract(t *testing.T) {
283283
ss := newTestSink(0*time.Second, nil)
284-
ss.extractor = func(key []string) ([]string, []metrics.Label, error) {
284+
ss.extractor = func(key []string, kind string) ([]string, []metrics.Label, error) {
285285
return key[:1], []metrics.Label{
286286
{
287287
Name: "method",

‎vault/vault.go‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ import "github.com/armon/go-metrics"
2121
// Extractor extracts known patterns from the key into metrics.Label for better metric grouping
2222
// and to help avoid the limit of 500 custom metric descriptors per project
2323
// (https://cloud.google.com/monitoring/quotas).
24-
func Extractor(key []string) ([]string, []metrics.Label, error) {
24+
func Extractor(key []string, kind string) ([]string, []metrics.Label, error) {
2525
// Metrics documented at https://www.vaultproject.io/docs/internals/telemetry.html should be
2626
// extracted here into a base metric name with appropriate labels extracted from the 'key'.
2727
switch len(key) {
2828
case 3: // metrics of format: *.*.*
2929
// vault.database.<method>
3030
if key[0] == "vault" && key[1] == "database" {
31-
return key[:2], []metrics.Label{
31+
return typeWrap(key[:2], kind), []metrics.Label{
3232
{
3333
Name: "method",
3434
Value: key[2],
@@ -37,14 +37,14 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
3737
}
3838
// vault.token.create_root
3939
if key[0] == "vault" && key[1] == "token" && key[2] == "create_root" {
40-
return key, nil, nil
40+
return typeWrap(key, kind), nil, nil
4141
}
4242

4343
// vault.barrier.<method>
4444
// vault.token.<method>
4545
// vault.policy.<method>
4646
if key[0] == "vault" && (key[1] == "barrier" || key[1] == "token" || key[1] == "policy") {
47-
return key[:2], []metrics.Label{
47+
return typeWrap(key[:2], kind), []metrics.Label{
4848
{
4949
Name: "method",
5050
Value: key[2],
@@ -53,7 +53,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
5353
}
5454
// vault.<backend>.<method>
5555
if key[0] == "vault" && (key[2] == "put" || key[2] == "get" || key[2] == "delete" || key[2] == "list") {
56-
return key[:2], []metrics.Label{
56+
return typeWrap(key[:2], kind), []metrics.Label{
5757
{
5858
Name: "method",
5959
Value: key[2],
@@ -63,7 +63,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
6363
case 4: // metrics of format: *.*.*.*
6464
// vault.route.<method>.<mount>
6565
if key[0] == "vault" && key[1] == "route" {
66-
return key[:2], []metrics.Label{
66+
return typeWrap(key[:2], kind), []metrics.Label{
6767
{
6868
Name: "method",
6969
Value: key[2],
@@ -76,7 +76,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
7676
}
7777
// vault.audit.<type>.*
7878
if key[0] == "vault" && key[1] == "audit" {
79-
return []string{"vault", "audit", key[3]}, []metrics.Label{
79+
return typeWrap([]string{"vault", "audit", key[3]}, kind), []metrics.Label{
8080
{
8181
Name: "type",
8282
Value: key[2],
@@ -85,7 +85,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
8585
}
8686
// vault.rollback.attempt.<mount>
8787
if key[0] == "vault" && key[1] == "rollback" && key[2] == "attempt" {
88-
return key[:3], []metrics.Label{
88+
return typeWrap(key[:3], kind), []metrics.Label{
8989
{
9090
Name: "mount",
9191
Value: key[3],
@@ -94,7 +94,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
9494
}
9595
// vault.<backend>.lock.<method>
9696
if key[0] == "vault" && key[2] == "lock" {
97-
return key[:3], []metrics.Label{
97+
return typeWrap(key[:3], kind), []metrics.Label{
9898
{
9999
Name: "method",
100100
Value: key[3],
@@ -104,7 +104,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
104104
// vault.database.<name>.<method>
105105
// note: there are vault.database.<method>.error counters. Those are handled separately.
106106
if key[0] == "vault" && key[1] == "database" && key[3] != "error" {
107-
return key[:2], []metrics.Label{
107+
return typeWrap(key[:2], kind), []metrics.Label{
108108
{
109109
Name: "name",
110110
Value: key[2],
@@ -117,7 +117,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
117117
}
118118
//ivault.database.<method>.error
119119
if key[0] == "vault" && key[1] == "database" && key[3] == "error" {
120-
return []string{"vault", "database", "error"}, []metrics.Label{
120+
return typeWrap([]string{"vault", "database", "error"}, kind), []metrics.Label{
121121
{
122122
Name: "method",
123123
Value: key[2],
@@ -127,7 +127,7 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
127127
case 5:
128128
// vault.database.<name>.<method>.error
129129
if key[0] == "vault" && key[1] == "database" && key[4] == "error" {
130-
return []string{key[0], key[1], key[4]}, []metrics.Label{
130+
return typeWrap([]string{key[0], key[1], key[4]}, kind), []metrics.Label{
131131
{
132132
Name: "name",
133133
Value: key[2],
@@ -141,7 +141,22 @@ func Extractor(key []string) ([]string, []metrics.Label, error) {
141141
default:
142142
// unknown key pattern, keep it as-is.
143143
}
144-
return key, nil, nil
144+
return typeWrap(key, kind), nil, nil
145+
}
146+
147+
func typeWrap(key []string, kind string) []string {
148+
out := []string{}
149+
for _, a := range key {
150+
out = append(out, a)
151+
}
152+
switch kind {
153+
case "counter":
154+
return append(out, kind)
155+
case "gauge":
156+
return append(out, kind)
157+
default:
158+
return out
159+
}
145160
}
146161

147162
// Bucketer specifies the bucket boundaries that should be used for the given metric key.

‎vault/vault_test.go‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1352,7 +1352,26 @@ func TestExtractor(t *testing.T) {
13521352

13531353
for _, tc := range tests {
13541354
t.Run(tc.desc, func(t *testing.T) {
1355-
key, labels, _ := Extractor(tc.in)
1355+
// counter
1356+
key, labels, _ := Extractor(tc.in, "counter")
1357+
if diff := cmp.Diff(append(tc.wantKey, "counter"), key); diff != "" {
1358+
t.Errorf("Extractor(%s) mismatch key (-want +got):\n%s", tc.in, diff)
1359+
}
1360+
if diff := cmp.Diff(tc.wantLabels, labels); diff != "" {
1361+
t.Errorf("Extractor(%s) mismatch labels (-want +got):\n%s", tc.in, diff)
1362+
}
1363+
1364+
// gauge
1365+
key, labels, _ = Extractor(tc.in, "gauge")
1366+
if diff := cmp.Diff(append(tc.wantKey, "gauge"), key); diff != "" {
1367+
t.Errorf("Extractor(%s) mismatch key (-want +got):\n%s", tc.in, diff)
1368+
}
1369+
if diff := cmp.Diff(tc.wantLabels, labels); diff != "" {
1370+
t.Errorf("Extractor(%s) mismatch labels (-want +got):\n%s", tc.in, diff)
1371+
}
1372+
1373+
// histogram
1374+
key, labels, _ = Extractor(tc.in, "histogram")
13561375
if diff := cmp.Diff(tc.wantKey, key); diff != "" {
13571376
t.Errorf("Extractor(%s) mismatch key (-want +got):\n%s", tc.in, diff)
13581377
}

0 commit comments

Comments
 (0)