Merge pull request kmesh-net#887 from skwwwwww/performance-monitoring

Add Performance Monitoring

Author: kmesh-bot

bpf/kmesh/probes/performance_probe.h

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/* Copyright Authors of Kmesh */
+
+#ifndef __KMESH_BPF_PERFORMANCE_MONITOR_H__
+#define __KMESH_BPF_PERFORMANCE_MONITOR_H__
+
+#define PERF_MONITOR 0
+
+#include "bpf_common.h"
+
+enum {
+    SOCK_TRAFFIC_CONTROL = 1,
+    XDP_SHUTDOWN = 2,
+    ENABLE_ENCODING_METADATA = 3,
+};
+
+struct operation_usage_data {
+    __u64 start_time;
+    __u64 end_time;
+    __u64 pid_tgid;
+    __u32 operation_type;
+};
+
+struct operation_usage_key {
+    __u64 socket_cookie;
+    __u32 operation_type;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __type(key, struct operation_usage_key);
+    __type(value, struct operation_usage_data);
+    __uint(map_flags, BPF_F_NO_PREALLOC);
+    __uint(max_entries, 131072);
+} kmesh_perf_map SEC(".maps");
+
+struct {
+    __uint(type, BPF_MAP_TYPE_RINGBUF);
+    __uint(max_entries, RINGBUF_SIZE);
+} kmesh_perf_info SEC(".maps");
+
+static inline void performance_report(struct operation_usage_data *data)
+{
+    struct operation_usage_data *info = NULL;
+    info = bpf_ringbuf_reserve(&kmesh_perf_info, sizeof(struct operation_usage_data), 0);
+    if (!info) {
+        BPF_LOG(ERR, PROBE, "bpf_ringbuf_reserve map proformance info failed\n");
+        return;
+    }
+    info->start_time = data->start_time;
+    info->end_time = data->end_time;
+    info->operation_type = data->operation_type;
+    info->pid_tgid = data->pid_tgid;
+    bpf_ringbuf_submit(info, 0);
+}
+
+static inline void observe_on_operation_start(__u32 operation_type, struct kmesh_context *kmesh_ctx)
+{
+#if PERF_MONITOR
+    struct operation_usage_data data = {};
+    struct operation_usage_key key = {};
+    struct bpf_sock_addr *ctx = kmesh_ctx->ctx;
+    __u64 socket_cookie = bpf_get_socket_cookie(ctx);
+    key.operation_type = operation_type;
+    key.socket_cookie = socket_cookie;
+    data.start_time = bpf_ktime_get_ns();
+    data.operation_type = operation_type;
+    bpf_map_update_elem(&kmesh_perf_map, &key, &data, BPF_ANY);
+    return;
+#else
+    return;
+#endif
+}
+
+static inline void observe_on_operation_end(__u32 operation_type, struct kmesh_context *kmesh_ctx)
+{
+#if PERF_MONITOR
+    struct operation_usage_key key = {};
+    __u64 pid_tgid = bpf_get_current_pid_tgid();
+    struct bpf_sock_addr *ctx = kmesh_ctx->ctx;
+    __u64 socket_cookie = bpf_get_socket_cookie(ctx);
+    key.operation_type = operation_type;
+    key.socket_cookie = socket_cookie;
+    struct operation_usage_data *data = NULL;
+    data = bpf_map_lookup_elem(&kmesh_perf_map, &key);
+    if (data) {
+        data->end_time = bpf_ktime_get_ns();
+        data->pid_tgid = pid_tgid;
+        performance_report(data);
+    }
+    bpf_map_delete_elem(&kmesh_perf_map, &key);
+    return;
+#else
+    return;
+#endif
+}
+#endif

bpf/kmesh/probes/probe.h

@@ -5,6 +5,7 @@
 #define __KMESH_BPF_PROBE_H__
 
 #include "tcp_probe.h"
+#include "performance_probe.h"
 
 static inline void observe_on_pre_connect(struct bpf_sock *sk)
 {

bpf/kmesh/workload/cgroup_sock.c

@@ -13,6 +13,7 @@
 
 static inline int sock_traffic_control(struct kmesh_context *kmesh_ctx)
 {
+    observe_on_operation_start(SOCK_TRAFFIC_CONTROL, kmesh_ctx);
     int ret;
     frontend_value *frontend_v = NULL;
     frontend_key frontend_k = {0};
@@ -61,7 +62,7 @@ static inline int sock_traffic_control(struct kmesh_context *kmesh_ctx)
             BPF_LOG(ERR, KMESH, "frontend_manager failed, ret:%d\n", ret);
         return ret;
     }
-
+    observe_on_operation_end(SOCK_TRAFFIC_CONTROL, kmesh_ctx);
     return 0;
 }
 

daemon/options/options.go

@@ -29,6 +29,7 @@ type BootstrapConfigs struct {
 	CniConfig           *cniConfig
 	ByPassConfig        *byPassConfig
 	SecretManagerConfig *secretConfig
+	PerfConfig          *perfConfig
 }
 
 func NewBootstrapConfigs() *BootstrapConfigs {
@@ -37,6 +38,7 @@ func NewBootstrapConfigs() *BootstrapConfigs {
 		CniConfig:           &cniConfig{},
 		ByPassConfig:        &byPassConfig{},
 		SecretManagerConfig: &secretConfig{},
+		PerfConfig:          &perfConfig{},
 	}
 }
 
@@ -54,6 +56,7 @@ func (c *BootstrapConfigs) AttachFlags(cmd *cobra.Command) {
 	c.CniConfig.AttachFlags(cmd)
 	c.ByPassConfig.AttachFlags(cmd)
 	c.SecretManagerConfig.AttachFlags(cmd)
+	c.PerfConfig.AttachFlags(cmd)
 }
 
 func (c *BootstrapConfigs) ParseConfigs() error {

daemon/options/perf.go

@@ -0,0 +1,29 @@
+/*
+ * Copyright The Kmesh Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package options
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type perfConfig struct {
+	EnablePerfMonitor bool
+}
+
+func (c *perfConfig) AttachFlags(cmd *cobra.Command) {
+	cmd.PersistentFlags().BoolVar(&c.EnablePerfMonitor, "enable-perfmonitor", false, "whether to start performance monitor or not, default to false")
+}

deploy/yaml/kmesh.yaml

@@ -74,7 +74,7 @@ spec:
           command: ["/bin/sh", "-c"]
           args:
             [
-              "./start_kmesh.sh --mode=dual-engine --enable-bypass=false --enable-bpf-log=true",
+              "./start_kmesh.sh --mode=dual-engine --enable-bypass=false --enable-bpf-log=true --enable-perfmonitor=false",
             ]
           securityContext:
             privileged: true

docs/proposal/performance_monitoring.md

@@ -0,0 +1,87 @@
+---
+title: proposal of Performance Monitoring
+authors:
+- "@skwwwwww" # Authors' GitHub accounts here.
+reviewers:
+- ""
+- TBD
+approvers:
+- ""
+- TBD
+
+creation-date: 2024-09-21
+
+---
+
+## proposal of Kmesh observability
+
+<!--
+This is the title of your KEP. Keep it short, simple, and descriptive. A good
+title can help communicate what the KEP is and should be considered as part of
+any review.
+-->
+### Summary
+
+<!--
+This section is incredibly important for producing high-quality, user-focused
+documentation such as release notes or a development roadmap.
+
+A good summary is probably at least a paragraph in length.
+-->
+
+The significance of performance monitoring in Kmesh as the foundation for efficient, scalable, and robust mesh systems is paramount. In Kmesh, monitoring key metrics such as Kmesh daemon activity, the number of eBPF maps, the number of entries in each map, and the duration of eBPF method execution is crucial to ensure optimal system performance.
+
+In this proposal, I will analyze the performance monitoring metrics of Kmesh. I will also suggest the implementation of enhanced observability features within Kmesh to capture these critical performance indicators. This will allow users to seamlessly monitor Kmesh's performance and ensure system efficiency.
+
+### Proposal
+
+Kmesh needs to collect metrics through the kernel and pass them on to the user mode. In the user mode, the data related to eBPF maps and operation duration can ultimately be queried through Prometheus and visualized using Grafana.
+
+#### Design Details
+
+##### metrics related operation duration
+
+This is because Kmesh needs to get metrics from the kernel and sent them to the user mode. We need a bpf map to record the metrics, as a vehicle for transferring.
+
+So we need to define a bpf map that contains all the required metrics:
+
+```
+struct operation_usage_data {
+    __u64 start_time;
+    __u64 end_time;
+    __u32 operation_type;
+};
+
+struct operation_usage_key {
+    __u32 tid;
+    __u32 operation_type;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __type(key, struct operation_usage_key);
+    __type(value, struct operation_usage_data);
+    __uint(max_entries, 1024);
+} kmehs_perf_map SEC(".maps");
+
+struct {
+    __uint(type, BPF_MAP_TYPE_RINGBUF);
+    __uint(max_entries, RINGBUF_SIZE);
+} kmesh_perf_info SEC(".maps");
+```
+
+collect the timestamps at the beginning and end of the function, and at the end of the function, write the data from the map into the ring buffer.
+
+##### metrics related ebpf map
+
+In user space, retrieve relevant information from the eBPF maps, including the total number of maps, the number of entries in each map, the maximum number of entries in the maps, and the memory space locked by the maps.
+
+![](pics/kmesh_map_and_operation_monitoring.jpg)
+
+##### metrics related Kmesh resource usage
+
+Provided by cAdvisor, use the `container_memory_usage_bytes` and `container_cpu_usage_seconds_total` metrics to visualize the memory usage and CPU consumption of Kmesh itself.
+![](pics/kmesh_daemon_monitoring.jpg)
+
+![](pics/performance_monitoring.jpg)
+