Merge pull request #1350 from hzxuzhonghu/cgroup_skb

Fix cgroup_skb/* get sk_storage failed

Author: kmesh-bot

bpf/include/bpf_common.h

@@ -260,6 +260,21 @@ static inline void remove_kmesh_managed_ip(__u32 family, __u32 ip4, __u32 *ip6)
         BPF_LOG(ERR, KMESH, "remove ip failed, err is %d\n", err);
 }
 
+static inline bool sock_conn_from_sim(struct __sk_buff *skb)
+{
+    __u16 dst_port = (__u16)(skb->remote_port >> 16);
+    if (bpf_ntohs(dst_port) != ENABLE_KMESH_PORT && bpf_ntohs(dst_port) != DISABLE_KMESH_PORT)
+        return false;
+
+    if (skb->protocol == AF_INET)
+        return bpf_ntohl(skb->remote_ip4) == CONTROL_CMD_IP;
+    // If directly read skb->remote_ip6. bpf prog load would fail with permission denied.
+    __u32 remote_ip6[4] = {0};
+    bpf_skb_load_bytes(skb, offsetof(struct __sk_buff, remote_ip6), &remote_ip6, sizeof(remote_ip6));
+    return (
+        remote_ip6[0] == 0 && remote_ip6[1] == 0 && remote_ip6[2] == 0 && bpf_ntohl(remote_ip6[3]) == CONTROL_CMD_IP);
+}
+
 static inline bool conn_from_sim(struct bpf_sock_ops *skops, __u32 ip, __u16 port)
 {
     __u16 remote_port = GET_SKOPS_REMOTE_PORT(skops);

bpf/kmesh/probes/probe.h

@@ -90,13 +90,15 @@ static inline void observe_on_data(struct bpf_sock *sk)
     struct sock_storage_data *storage = NULL;
     if (!sk)
         return;
+
     tcp_sock = bpf_tcp_sock(sk);
     if (!tcp_sock)
         return;
 
-    storage = bpf_sk_storage_get(&map_of_sock_storage, sk, 0, 0);
+    // Use BPF_LOCAL_STORAGE_GET_F_CREATE in case a connection being established before kmesh start.
+    storage = bpf_sk_storage_get(&map_of_sock_storage, sk, 0, BPF_LOCAL_STORAGE_GET_F_CREATE);
     if (!storage) {
-        BPF_LOG(ERR, PROBE, "on data: bpf_sk_storage_get failed\n");
+        BPF_LOG(ERR, PROBE, "on data: bpf_sk_storage_get failed dst  %u \n", bpf_ntohs(sk->dst_port));
         return;
     }
     __u64 now = bpf_ktime_get_ns();

bpf/kmesh/workload/cgroup_skb.c

@@ -24,8 +24,13 @@ int cgroup_skb_ingress_prog(struct __sk_buff *skb)
     if (!sk)
         return SK_PASS;
 
+    if (sock_conn_from_sim(skb)) {
+        return SK_PASS;
+    }
+
     if (!is_managed_by_kmesh_skb(skb))
         return SK_PASS;
+
     observe_on_data(sk);
     return SK_PASS;
 }
@@ -43,8 +48,13 @@ int cgroup_skb_egress_prog(struct __sk_buff *skb)
     if (!sk)
         return SK_PASS;
 
+    if (sock_conn_from_sim(skb)) {
+        return SK_PASS;
+    }
+
     if (!is_managed_by_kmesh_skb(skb))
         return SK_PASS;
+
     observe_on_data(sk);
     return SK_PASS;
 }