[Feature] Imporved metrics (#852)

Author: ajanikow

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.44.1
 	github.com/prometheus-operator/prometheus-operator/pkg/client v0.0.0-00010101000000-000000000000
 	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_model v0.2.0
 	github.com/robfig/cron v1.2.0
 	github.com/rs/zerolog v1.19.0
 	github.com/spf13/cobra v1.2.1

pkg/deployment/agency/cache.go

@@ -90,24 +90,24 @@ func (c *cache) Reload(ctx context.Context, client agency.Agency) (uint64, error
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	c.valid = false
-
 	cfg, err := getAgencyConfig(ctx, client)
 	if err != nil {
-		return cfg.CommitIndex, err
+		c.valid = false
+		return 0, err
 	}
 
-	if cfg.CommitIndex == c.commitIndex {
+	if cfg.CommitIndex == c.commitIndex && c.valid {
 		// We are on same index, nothing to do
 		return cfg.CommitIndex, err
 	}
 
 	if data, err := loadState(ctx, client); err != nil {
+		c.valid = false
 		return cfg.CommitIndex, err
 	} else {
 		c.data = data
 		c.valid = true
 		c.commitIndex = cfg.CommitIndex
-		return cfg.CommitIndex, err
+		return cfg.CommitIndex, nil
 	}
 }

pkg/deployment/agency/plan_collections.go

@@ -43,9 +43,18 @@ func (a StatePlanDBCollections) IsDBServerInCollections(name string) bool {
 }
 
 type StatePlanCollection struct {
+	Name   *string        `json:"name"`
 	Shards StatePlanShard `json:"shards"`
 }
 
+func (a StatePlanCollection) GetName(d string) string {
+	if a.Name == nil {
+		return d
+	}
+
+	return *a.Name
+}
+
 func (a StatePlanCollection) IsDBServerInShards(name string) bool {
 	for _, dbservers := range a.Shards {
 		for _, dbserver := range dbservers {

pkg/deployment/deployment.go

@@ -226,6 +226,8 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De
 		d.status.last.AcceptedSpec = apiObject.Spec.DeepCopy()
 	}
 
+	localInventory.Add(d)
+
 	go d.run()
 	go d.listenForPodEvents(d.stopCh)
 	go d.listenForPVCEvents(d.stopCh)

pkg/deployment/metrics.go

@@ -17,12 +17,122 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Ewout Prangsma
-//
 
 package deployment
 
+import (
+	"sync"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/metrics"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
 const (
 	// Component name for metrics of this package
 	metricsComponent = "deployment"
 )
+
+func init() {
+	localInventory = inventory{
+		deployments:                    map[string]map[string]*Deployment{},
+		deploymentsMetric:              metrics.NewDescription("arangodb_operator_deployments", "Number of active deployments", []string{"namespace", "deployment"}, nil),
+		deploymentMetricsMembersMetric: metrics.NewDescription("arango_operator_deployment_members", "List of members", []string{"namespace", "deployment", "role", "id"}, nil),
+		deploymentAgencyStateMetric:    metrics.NewDescription("arango_operator_deployment_agency_state", "Reachability of agency", []string{"namespace", "deployment"}, nil),
+		deploymentShardLeadersMetric:   metrics.NewDescription("arango_operator_deployment_shard_leaders", "Deployment leader shards distribution", []string{"namespace", "deployment", "database", "collection", "shard", "server"}, nil),
+		deploymentShardsMetric:         metrics.NewDescription("arango_operator_deployment_shards", "Deployment shards distribution", []string{"namespace", "deployment", "database", "collection", "shard", "server"}, nil),
+	}
+
+	prometheus.MustRegister(&localInventory)
+}
+
+var localInventory inventory
+
+var _ prometheus.Collector = &inventory{}
+
+type inventory struct {
+	lock        sync.Mutex
+	deployments map[string]map[string]*Deployment
+
+	deploymentsMetric, deploymentMetricsMembersMetric, deploymentAgencyStateMetric, deploymentShardsMetric, deploymentShardLeadersMetric metrics.Description
+}
+
+func (i *inventory) Describe(descs chan<- *prometheus.Desc) {
+	i.lock.Lock()
+	defer i.lock.Unlock()
+
+	metrics.NewPushDescription(descs).Push(i.deploymentsMetric, i.deploymentMetricsMembersMetric, i.deploymentAgencyStateMetric, i.deploymentShardLeadersMetric, i.deploymentShardsMetric)
+}
+
+func (i *inventory) Collect(m chan<- prometheus.Metric) {
+	i.lock.Lock()
+	defer i.lock.Unlock()
+
+	p := metrics.NewPushMetric(m)
+	for _, deployments := range i.deployments {
+		for _, deployment := range deployments {
+			p.Push(i.deploymentsMetric.Gauge(1, deployment.GetNamespace(), deployment.GetName()))
+
+			spec := deployment.GetSpec()
+			status, _ := deployment.GetStatus()
+
+			for _, member := range status.Members.AsList() {
+				p.Push(i.deploymentMetricsMembersMetric.Gauge(1, deployment.GetNamespace(), deployment.GetName(), member.Group.AsRole(), member.Member.ID))
+			}
+
+			if spec.Mode.Get().HasAgents() {
+				agency, agencyOk := deployment.GetAgencyCache()
+				if !agencyOk {
+					p.Push(i.deploymentAgencyStateMetric.Gauge(0, deployment.GetNamespace(), deployment.GetName()))
+					continue
+				}
+
+				p.Push(i.deploymentAgencyStateMetric.Gauge(1, deployment.GetNamespace(), deployment.GetName()))
+
+				if spec.Mode.Get() == api.DeploymentModeCluster {
+					for db, collections := range agency.Current.Collections {
+						for collection, shards := range collections {
+							for shard, details := range shards {
+								for id, server := range details.Servers {
+									name := "UNKNOWN"
+									if _, ok := agency.Plan.Collections[db]; ok {
+										if _, ok := agency.Plan.Collections[db][collection]; ok {
+											name = agency.Plan.Collections[db][collection].GetName(name)
+										}
+									}
+
+									m := []string{
+										deployment.GetNamespace(),
+										deployment.GetName(),
+										db,
+										name,
+										shard,
+										server,
+									}
+
+									if id == 0 {
+										p.Push(i.deploymentShardLeadersMetric.Gauge(1, m...))
+									}
+									p.Push(i.deploymentShardsMetric.Gauge(1, m...))
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func (i *inventory) Add(d *Deployment) {
+	i.lock.Lock()
+	defer i.lock.Unlock()
+
+	name, namespace := d.GetName(), d.GetNamespace()
+
+	if _, ok := i.deployments[namespace]; !ok {
+		i.deployments[namespace] = map[string]*Deployment{}
+	}
+
+	i.deployments[namespace][name] = d
+}

pkg/util/metrics/collector.go

@@ -0,0 +1,27 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+type Collector interface {
+	Collect(description Description, metrics chan<- prometheus.Metric)
+}

pkg/util/metrics/description.go

@@ -0,0 +1,90 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metrics
+
+import (
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+type Description interface {
+	Desc() *prometheus.Desc
+	Labels(labels ...string) []*dto.LabelPair
+
+	Collect(out chan<- prometheus.Metric, collect func(p Producer) error)
+
+	Gauge(value float64, labels ...string) Metric
+}
+
+type description struct {
+	variableLabels []string
+	constLabels    prometheus.Labels
+	desc           *prometheus.Desc
+}
+
+func (d *description) Gauge(value float64, labels ...string) Metric {
+	return newGauge(d, value, labels...)
+}
+
+func (d *description) Collect(out chan<- prometheus.Metric, collect func(p Producer) error) {
+	if err := collect(newProducer(out, d)); err != nil {
+		out <- newError(d, err)
+	}
+}
+
+func (d *description) Desc() *prometheus.Desc {
+	return d.desc
+}
+
+func (d *description) Labels(labels ...string) []*dto.LabelPair {
+	var l []*dto.LabelPair
+
+	for k, v := range d.constLabels {
+		var z dto.LabelPair
+
+		z.Name = util.NewString(k)
+		z.Value = util.NewString(v)
+		l = append(l, &z)
+	}
+
+	for id := range labels {
+		if id >= len(d.variableLabels) {
+			break
+		}
+
+		var z dto.LabelPair
+
+		z.Name = util.NewString(d.variableLabels[id])
+		z.Value = util.NewString(labels[id])
+		l = append(l, &z)
+	}
+
+	return l
+}
+
+func NewDescription(fqName, help string, variableLabels []string, constLabels prometheus.Labels) Description {
+	return &description{
+		variableLabels: variableLabels,
+		constLabels:    constLabels,
+		desc:           prometheus.NewDesc(fqName, help, variableLabels, constLabels),
+	}
+}

pkg/util/metrics/metric.go

@@ -0,0 +1,88 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+type Metric interface {
+	prometheus.Metric
+}
+
+type Gauge interface {
+	Metric
+}
+
+type Error interface {
+	Metric
+}
+
+type errorRet struct {
+	desc Description
+	err  error
+}
+
+func (e errorRet) Desc() *prometheus.Desc {
+	return e.desc.Desc()
+}
+
+func (e errorRet) Write(metric *dto.Metric) error {
+	return e.err
+}
+
+func newError(desc Description, value error) Error {
+	return errorRet{
+		desc: desc,
+		err:  value,
+	}
+}
+
+func newGauge(desc Description, value float64, labels ...string) Gauge {
+	return gauge{
+		desc:   desc,
+		labels: labels,
+		value:  value,
+	}
+}
+
+type gauge struct {
+	desc Description
+
+	labels []string
+
+	value float64
+}
+
+func (g gauge) Desc() *prometheus.Desc {
+	return g.desc.Desc()
+}
+
+func (g gauge) Write(metric *dto.Metric) error {
+	metric.Label = g.desc.Labels(g.labels...)
+
+	metric.Gauge = &dto.Gauge{
+		Value: &g.value,
+	}
+
+	return nil
+}