Skip to content

Commit f1b1696

Browse files
tianx666tianx666
committed
libxscale: Introduce xscale user space RDMA provider
libxscale is a user-space driver that provides RDMA capabilities to user applications. The current patch includes the following components: 1. basic compile framework 2. register/unregister user-space driver via verbs 3. query_port 4. query_device_ex Signed-off-by: Tian Xin <tianx@yunsilicon.com> Signed-off-by: Wei Honggang <weihg@yunsilicon.com> Signed-off-by: Zhao Qianwei <zhaoqw@yunsilicon.com> Signed-off-by: Li Qiang <liq@yunsilicon.com> Signed-off-by: Yan Lei <jacky@yunsilicon.com>
1 parent dd29f7c commit f1b1696

File tree

7 files changed

+1116
-0
lines changed

7 files changed

+1116
-0
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,7 @@ add_subdirectory(providers/mthca)
729729
add_subdirectory(providers/ocrdma)
730730
add_subdirectory(providers/qedr)
731731
add_subdirectory(providers/vmw_pvrdma)
732+
add_subdirectory(providers/xscale)
732733
endif()
733734

734735
add_subdirectory(providers/hfi1verbs)

‎MAINTAINERS‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,12 @@ PYVERBS
189189
M: Edward Srouji <edwards@mellanox.com>
190190
S: Supported
191191
F: pyverbs/
192+
193+
MLX5 USERSPACE PROVIDER (for xsc_ib.ko)
194+
M: Wei Honggang <weihg@yunsilicon.com>
195+
M: Zhao Qianwei <zhaoqw@yunsilicon.com>
196+
M: Li Qiang <liq@yunsilicon.com>
197+
M: Tian Xin <tianx@yunsilicon.com>
198+
M: Yan Lei <jacky@yunsilicon.com>
199+
S: Supported
200+
F: providers/xscale/

‎providers/xscale/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
rdma_provider(xscale
2+
xscale.c
3+
verbs.c
4+
)

‎providers/xscale/verbs.c‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
/*
3+
* Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd.
4+
* All rights reserved.
5+
*/
6+
7+
#include <config.h>
8+
9+
#include <stdlib.h>
10+
#include <stdio.h>
11+
#include <stdatomic.h>
12+
#include <string.h>
13+
#include <pthread.h>
14+
#include <errno.h>
15+
#include <limits.h>
16+
#include <sys/types.h>
17+
#include <sys/stat.h>
18+
#include <fcntl.h>
19+
#include <unistd.h>
20+
#include <sys/mman.h>
21+
#include <ccan/array_size.h>
22+
23+
#include <util/compiler.h>
24+
#include <util/mmio.h>
25+
#include <rdma/ib_user_ioctl_cmds.h>
26+
#include <rdma/xsc_user_ioctl_cmds.h>
27+
#include <infiniband/cmd_write.h>
28+
29+
#include "xscale.h"
30+
#include "xsc-abi.h"
31+
32+
int xsc_single_threaded;
33+
34+
int xsc_query_port(struct ibv_context *context, u8 port,
35+
struct ibv_port_attr *attr)
36+
{
37+
struct ibv_query_port cmd;
38+
39+
return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
40+
}
41+
42+
static void xsc_set_fw_version(struct ibv_device_attr *attr,
43+
union xsc_ib_fw_ver *fw_ver)
44+
{
45+
u8 ver_major = fw_ver->s.ver_major;
46+
u8 ver_minor = fw_ver->s.ver_minor;
47+
u16 ver_patch = fw_ver->s.ver_patch;
48+
u32 ver_tweak = fw_ver->s.ver_tweak;
49+
50+
if (ver_tweak == 0) {
51+
snprintf(attr->fw_ver, sizeof(attr->fw_ver), "v%u.%u.%u",
52+
ver_major, ver_minor, ver_patch);
53+
} else {
54+
snprintf(attr->fw_ver, sizeof(attr->fw_ver), "v%u.%u.%u+%u",
55+
ver_major, ver_minor, ver_patch, ver_tweak);
56+
}
57+
}
58+
59+
int xsc_query_device_ex(struct ibv_context *context,
60+
const struct ibv_query_device_ex_input *input,
61+
struct ibv_device_attr_ex *attr, size_t attr_size)
62+
{
63+
struct xsc_context *xctx = to_xctx(context);
64+
struct xsc_query_device_ex_resp resp = {};
65+
size_t resp_size =
66+
(xctx->cmds_supp_uhw & XSC_USER_CMDS_SUPP_UHW_QUERY_DEVICE) ?
67+
sizeof(resp) :
68+
sizeof(resp.ibv_resp);
69+
struct ibv_device_attr *a;
70+
union xsc_ib_fw_ver raw_fw_ver;
71+
int err;
72+
73+
raw_fw_ver.data = 0;
74+
err = ibv_cmd_query_device_any(context, input, attr, attr_size,
75+
&resp.ibv_resp, &resp_size);
76+
if (err)
77+
return err;
78+
79+
if (attr_size >= offsetofend(struct ibv_device_attr_ex, tso_caps)) {
80+
attr->tso_caps.max_tso = resp.tso_caps.max_tso;
81+
attr->tso_caps.supported_qpts = resp.tso_caps.supported_qpts;
82+
}
83+
if (attr_size >= offsetofend(struct ibv_device_attr_ex, rss_caps)) {
84+
attr->rss_caps.rx_hash_fields_mask =
85+
resp.rss_caps.rx_hash_fields_mask;
86+
attr->rss_caps.rx_hash_function =
87+
resp.rss_caps.rx_hash_function;
88+
}
89+
if (attr_size >=
90+
offsetofend(struct ibv_device_attr_ex, packet_pacing_caps)) {
91+
attr->packet_pacing_caps.qp_rate_limit_min =
92+
resp.packet_pacing_caps.qp_rate_limit_min;
93+
attr->packet_pacing_caps.qp_rate_limit_max =
94+
resp.packet_pacing_caps.qp_rate_limit_max;
95+
attr->packet_pacing_caps.supported_qpts =
96+
resp.packet_pacing_caps.supported_qpts;
97+
}
98+
99+
if (resp.xsc_ib_support_multi_pkt_send_wqes & XSC_IB_ALLOW_MPW)
100+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_MPW_ALLOWED;
101+
102+
if (resp.xsc_ib_support_multi_pkt_send_wqes & XSC_IB_SUPPORT_EMPW)
103+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_ENHANCED_MPW;
104+
105+
xctx->tunnel_offloads_caps = resp.tunnel_offloads_caps;
106+
xctx->packet_pacing_caps = resp.packet_pacing_caps;
107+
108+
if (resp.flags & XSC_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP)
109+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_CQE_128B_COMP;
110+
111+
if (resp.flags & XSC_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD)
112+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_CQE_128B_PAD;
113+
114+
raw_fw_ver.data = resp.ibv_resp.base.fw_ver;
115+
a = &attr->orig_attr;
116+
xsc_set_fw_version(a, &raw_fw_ver);
117+
118+
return 0;
119+
}

‎providers/xscale/xsc-abi.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd.
4+
* All rights reserved.
5+
*/
6+
7+
#ifndef XSC_ABI_H
8+
#define XSC_ABI_H
9+
10+
#include <infiniband/kern-abi.h>
11+
#include <infiniband/verbs.h>
12+
#include <rdma/xsc-abi.h>
13+
#include <kernel-abi/xsc-abi.h>
14+
15+
#define XSC_UVERBS_MIN_ABI_VERSION 1
16+
#define XSC_UVERBS_MAX_ABI_VERSION 1
17+
18+
DECLARE_DRV_CMD(xsc_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
19+
xsc_ib_alloc_ucontext_req, xsc_ib_alloc_ucontext_resp);
20+
DECLARE_DRV_CMD(xsc_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty,
21+
xsc_ib_alloc_pd_resp);
22+
DECLARE_DRV_CMD(xsc_create_cq, IB_USER_VERBS_CMD_CREATE_CQ, xsc_ib_create_cq,
23+
xsc_ib_create_cq_resp);
24+
DECLARE_DRV_CMD(xsc_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ,
25+
xsc_ib_create_cq, xsc_ib_create_cq_resp);
26+
DECLARE_DRV_CMD(xsc_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
27+
xsc_ib_create_qp, xsc_ib_create_qp_resp);
28+
DECLARE_DRV_CMD(xsc_create_qp, IB_USER_VERBS_CMD_CREATE_QP, xsc_ib_create_qp,
29+
xsc_ib_create_qp_resp);
30+
DECLARE_DRV_CMD(xsc_query_device_ex, IB_USER_VERBS_EX_CMD_QUERY_DEVICE, empty,
31+
xsc_ib_query_device_resp);
32+
DECLARE_DRV_CMD(xsc_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP, empty,
33+
xsc_ib_modify_qp_resp);
34+
35+
struct xsc_modify_qp {
36+
struct ibv_modify_qp_ex ibv_cmd;
37+
__u32 comp_mask;
38+
struct xsc_ib_burst_info burst_info;
39+
__u32 reserved;
40+
};
41+
42+
#endif /* XSC_ABI_H */

0 commit comments

Comments
 (0)