Added prometheus metrics (#366)

Author: neufeldtech

CHANGELOG.md

@@ -7,6 +7,7 @@
 * Improve server startup message (#358, @areveny)
 * Introduce yaml linter. (#362, @miry)
 * Handle slicer toxic with zero `SizeVariation` and fix slicing randomization (#359, @areveny)
+* Added /metrics endpoint for exposing Prometheus-compatible internal metrics (#366, @neufeldtech)
 
 # [2.3.0] - 2021-12-23
 

METRICS.md

@@ -0,0 +1,44 @@
+# Metrics
+
+- [Metrics](#metrics)
+    - [Runtime Metrics](#runtime-metrics)
+    - [Proxy Metrics](#proxy-metrics)
+      - [toxiproxy_proxy_received_bytes_total / toxiproxy_proxy_sent_bytes_total](#toxiproxy_proxy_received_bytes_total--toxiproxy_proxy_sent_bytes_total)
+
+### Runtime Metrics
+
+To enable runtime metrics related to the state of the go runtime, build version, process info, use the `-runtime-metrics` flag.
+
+For more details, see below:
+- [NewGoCollector](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/collectors#NewGoCollector)
+- [NewBuildInfoCollector](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/collectors#NewBuildInfoCollector)
+- [NewProcessCollector](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/collectors#NewProcessCollector)
+
+### Proxy Metrics
+
+To enable metrics related to toxiproxy internals, use the `-proxy-metrics` flag.
+#### toxiproxy_proxy_received_bytes_total / toxiproxy_proxy_sent_bytes_total
+
+The total number of bytes received/sent on a given proxy link in a given direction
+
+```mermaid
+sequenceDiagram
+    Client->>+Toxiproxy: toxiproxy_proxy_received_bytes_total{direction="upstream"}
+    Toxiproxy->>+Server: toxiproxy_proxy_sent_bytes_total{direction="upstream"}
+    Server->>+Toxiproxy: toxiproxy_proxy_received_bytes_total{direction="downstream"}
+    Toxiproxy->>+Client: toxiproxy_proxy_sent_bytes_total{direction="downstream"}
+```
+
+**Type**
+
+Counter
+
+**Labels**
+
+| Label     | Description                    | Example               |
+|-----------|--------------------------------|-----------------------|
+| direction | Direction of the link          | upstream / downstream |
+| listener  | Listener address of this proxy | 0.0.0.0:8080          |
+| proxy     | Proxy name                     | my-proxy              |
+| upstream  | Upstream address of this proxy | httpbin.org:80        |
+

README.md

@@ -41,31 +41,36 @@ stopping you from creating a client in any other language (see
 
 ## Table of Contents
 
-1. [Why yet another chaotic TCP proxy?](#why-yet-another-chaotic-tcp-proxy)
-2. [Clients](#clients)
-3. [Example](#example)
-4. [Usage](#usage)
-   1. [Installing](#1-installing-toxiproxy)
-      1. [Upgrading from 1.x](#upgrading-from-toxiproxy-1x)
-   2. [Populating](#2-populating-toxiproxy)
-   3. [Using](#3-using-toxiproxy)
-   4. [Logging](#4-logging)
-5. [Toxics](#toxics)
-   1. [Latency](#latency)
-   2. [Down](#down)
-   3. [Bandwidth](#bandwidth)
-   4. [Slow close](#slow_close)
-   5. [Timeout](#timeout)
-   6. [Reset peer](#reset_peer)
-   7. [Slicer](#slicer)
-6. [HTTP API](#http-api)
-   1. [Proxy fields](#proxy-fields)
-   2. [Toxic fields](#toxic-fields)
-   3. [Endpoints](#endpoints)
-   4. [Populating Proxies](#populating-proxies)
-7. [CLI example](#cli-example)
-8. [FAQ](#frequently-asked-questions)
-9. [Development](#development)
+- [Toxiproxy](#toxiproxy)
+  - [Table of Contents](#table-of-contents)
+  - [Why yet another chaotic TCP proxy?](#why-yet-another-chaotic-tcp-proxy)
+  - [Clients](#clients)
+  - [Example](#example)
+  - [Usage](#usage)
+    - [1. Installing Toxiproxy](#1-installing-toxiproxy)
+      - [Upgrading from Toxiproxy 1.x](#upgrading-from-toxiproxy-1x)
+    - [2. Populating Toxiproxy](#2-populating-toxiproxy)
+    - [3. Using Toxiproxy](#3-using-toxiproxy)
+    - [4. Logging](#4-logging)
+    - [Toxics](#toxics)
+      - [latency](#latency)
+      - [down](#down)
+      - [bandwidth](#bandwidth)
+      - [slow_close](#slow_close)
+      - [timeout](#timeout)
+      - [reset_peer](#reset_peer)
+      - [slicer](#slicer)
+      - [limit_data](#limit_data)
+    - [HTTP API](#http-api)
+      - [Proxy fields:](#proxy-fields)
+      - [Toxic fields:](#toxic-fields)
+      - [Endpoints](#endpoints)
+      - [Populating Proxies](#populating-proxies)
+    - [CLI Example](#cli-example)
+    - [Metrics](#metrics)
+    - [Frequently Asked Questions](#frequently-asked-questions)
+    - [Development](#development)
+    - [Release](#release)
 
 ## Why yet another chaotic TCP proxy?
 
@@ -497,6 +502,7 @@ All endpoints are JSON.
  - **DELETE /proxies/{proxy}/toxics/{toxic}** - Remove an active toxic
  - **POST /reset** - Enable all proxies and remove all active toxics
  - **GET /version** - Returns the server version number
+ - **GET /metrics** - Returns Prometheus-compatible metrics
 
 #### Populating Proxies
 
@@ -565,6 +571,11 @@ $ redis-cli -p 26379
 Could not connect to Redis at 127.0.0.1:26379: Connection refused
 ```
 
+### Metrics
+
+Toxiproxy exposes Prometheus-compatible metrics via its HTTP API at /metrics.
+See [METRICS.md](./METRICS.md) for full descriptions
+
 ### Frequently Asked Questions
 
 **How fast is Toxiproxy?** The speed of Toxiproxy depends largely on your hardware,

api.go

@@ -16,11 +16,13 @@ import (
 
 type ApiServer struct {
 	Collection *ProxyCollection
+	Metrics    *metricsContainer
 }
 
-func NewServer() *ApiServer {
+func NewServer(m *metricsContainer) *ApiServer {
 	return &ApiServer{
 		Collection: NewProxyCollection(),
+		Metrics:    m,
 	}
 }
 
@@ -34,7 +36,7 @@ func (server *ApiServer) PopulateConfig(filename string) {
 		return
 	}
 
-	proxies, err := server.Collection.PopulateJson(file)
+	proxies, err := server.Collection.PopulateJson(server, file)
 	if err != nil {
 		logrus.WithFields(logrus.Fields{
 			"config": filename,
@@ -75,6 +77,10 @@ func (server *ApiServer) Listen(host string, port string) {
 
 	r.HandleFunc("/version", server.Version).Methods("GET")
 
+	if server.Metrics.anyMetricsEnabled() {
+		r.Handle("/metrics", server.Metrics.handler())
+	}
+
 	http.Handle("/", StopBrowsersMiddleware(r))
 
 	logrus.WithFields(logrus.Fields{
@@ -145,7 +151,7 @@ func (server *ApiServer) ProxyCreate(response http.ResponseWriter, request *http
 		return
 	}
 
-	proxy := NewProxy()
+	proxy := NewProxy(server)
 	proxy.Name = input.Name
 	proxy.Listen = input.Listen
 	proxy.Upstream = input.Upstream
@@ -169,7 +175,7 @@ func (server *ApiServer) ProxyCreate(response http.ResponseWriter, request *http
 }
 
 func (server *ApiServer) Populate(response http.ResponseWriter, request *http.Request) {
-	proxies, err := server.Collection.PopulateJson(request.Body)
+	proxies, err := server.Collection.PopulateJson(server, request.Body)
 
 	apiErr, ok := err.(*ApiError)
 	if !ok && err != nil {

api_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/Shopify/toxiproxy/v2"
 	tclient "github.com/Shopify/toxiproxy/v2/client"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var testServer *toxiproxy.ApiServer
@@ -19,7 +20,8 @@ func WithServer(t *testing.T, f func(string)) {
 	// Make sure only one server is running at a time. Apparently there's no clean
 	// way to shut it down between each test run.
 	if testServer == nil {
-		testServer = toxiproxy.NewServer()
+		testServer = toxiproxy.NewServer(toxiproxy.NewMetricsContainer(prometheus.NewRegistry()))
+
 		go testServer.Listen("localhost", "8475")
 
 		// Allow server to start. There's no clean way to know when it listens.

cmd/server/server.go

@@ -10,17 +10,21 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
 	"github.com/Shopify/toxiproxy/v2"
+	"github.com/Shopify/toxiproxy/v2/collectors"
 )
 
 type cliArguments struct {
-	host         string
-	port         string
-	config       string
-	seed         int64
-	printVersion bool
+	host           string
+	port           string
+	config         string
+	seed           int64
+	printVersion   bool
+	proxyMetrics   bool
+	runtimeMetrics bool
 }
 
 func parseArguments() cliArguments {
@@ -33,6 +37,10 @@ func parseArguments() cliArguments {
 		"JSON file containing proxies to create on startup")
 	flag.Int64Var(&result.seed, "seed", time.Now().UTC().UnixNano(),
 		"Seed for randomizing toxics with")
+	flag.BoolVar(&result.runtimeMetrics, "runtime-metrics", false,
+		`enable runtime-related prometheus metrics (default "false")`)
+	flag.BoolVar(&result.proxyMetrics, "proxy-metrics", false,
+		`enable toxiproxy-specific prometheus metrics (default "false")`)
 	flag.BoolVar(&result.printVersion, "version", false,
 		`print the version (default "false")`)
 	flag.Parse()
@@ -63,7 +71,14 @@ func run(cli cliArguments) {
 
 	rand.Seed(cli.seed)
 
-	server := toxiproxy.NewServer()
+	metrics := toxiproxy.NewMetricsContainer(prometheus.NewRegistry())
+	server := toxiproxy.NewServer(metrics)
+	if cli.proxyMetrics {
+		server.Metrics.ProxyMetrics = collectors.NewProxyMetricCollectors()
+	}
+	if cli.runtimeMetrics {
+		server.Metrics.RuntimeMetrics = collectors.NewRuntimeMetricCollectors()
+	}
 	if len(cli.config) > 0 {
 		server.PopulateConfig(cli.config)
 	}

collectors/common.go

@@ -0,0 +1,5 @@
+package collectors
+
+const (
+	namespace string = "toxiproxy"
+)

collectors/proxy.go

@@ -0,0 +1,46 @@
+package collectors
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type ProxyMetricCollectors struct {
+	collectors  []prometheus.Collector
+	proxyLabels []string
+
+	ReceivedBytesTotal *prometheus.CounterVec
+	SentBytesTotal     *prometheus.CounterVec
+}
+
+func (c *ProxyMetricCollectors) Collectors() []prometheus.Collector {
+	return c.collectors
+}
+
+func NewProxyMetricCollectors() *ProxyMetricCollectors {
+	var m ProxyMetricCollectors
+	m.proxyLabels = []string{
+		"direction",
+		"proxy",
+		"listener",
+		"upstream",
+	}
+	m.ReceivedBytesTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: "proxy",
+			Name:      "received_bytes_total",
+		},
+		m.proxyLabels)
+	m.collectors = append(m.collectors, m.ReceivedBytesTotal)
+
+	m.SentBytesTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: "proxy",
+			Name:      "sent_bytes_total",
+		},
+		m.proxyLabels)
+	m.collectors = append(m.collectors, m.SentBytesTotal)
+
+	return &m
+}

collectors/runtime.go

@@ -0,0 +1,23 @@
+package collectors
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+)
+
+type RuntimeMetricCollectors struct {
+	collectors []prometheus.Collector
+}
+
+func (c *RuntimeMetricCollectors) Collectors() []prometheus.Collector {
+	return c.collectors
+}
+
+func NewRuntimeMetricCollectors() *RuntimeMetricCollectors {
+	var m RuntimeMetricCollectors
+	m.collectors = append(m.collectors, collectors.NewGoCollector())
+	m.collectors = append(m.collectors, collectors.NewBuildInfoCollector())
+	m.collectors = append(m.collectors,
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+	return &m
+}

go.mod

@@ -4,15 +4,24 @@ go 1.17
 
 require (
 	github.com/gorilla/mux v1.8.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/urfave/cli/v2 v2.3.0
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
-	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
 )