Skip to content

Commit a4c4e7d

Browse files
tianx666tianx666
committed
libxscale: Introduce xscale user space RDMA provider
libxscale is a user-space driver that provides RDMA capabilities to user applications. The current patch includes the following components: 1. basic compile framework 2. register/unregister user-space driver via verbs 3. query_port 4. query_device_ex Signed-off-by: Tian Xin <tianx@yunsilicon.com> Signed-off-by: Wei Honggang <weihg@yunsilicon.com> Signed-off-by: Zhao Qianwei <zhaoqw@yunsilicon.com> Signed-off-by: Li Qiang <liq@yunsilicon.com> Signed-off-by: Yan Lei <jacky@yunsilicon.com>
1 parent dd29f7c commit a4c4e7d

File tree

7 files changed

+1050
-0
lines changed

7 files changed

+1050
-0
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,7 @@ add_subdirectory(providers/mthca)
729729
add_subdirectory(providers/ocrdma)
730730
add_subdirectory(providers/qedr)
731731
add_subdirectory(providers/vmw_pvrdma)
732+
add_subdirectory(providers/xscale)
732733
endif()
733734

734735
add_subdirectory(providers/hfi1verbs)

‎MAINTAINERS‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,12 @@ PYVERBS
189189
M: Edward Srouji <edwards@mellanox.com>
190190
S: Supported
191191
F: pyverbs/
192+
193+
XSCALE USERSPACE PROVIDER (for xsc_ib.ko)
194+
M: Wei Honggang <weihg@yunsilicon.com>
195+
M: Zhao Qianwei <zhaoqw@yunsilicon.com>
196+
M: Li Qiang <liq@yunsilicon.com>
197+
M: Tian Xin <tianx@yunsilicon.com>
198+
M: Yan Lei <jacky@yunsilicon.com>
199+
S: Supported
200+
F: providers/xscale/

‎providers/xscale/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
rdma_provider(xscale
2+
xscale.c
3+
verbs.c
4+
)

‎providers/xscale/verbs.c‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
/*
3+
* Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd.
4+
* All rights reserved.
5+
*/
6+
7+
#include <config.h>
8+
9+
#include <stdlib.h>
10+
#include <stdio.h>
11+
#include <stdatomic.h>
12+
#include <string.h>
13+
#include <pthread.h>
14+
#include <errno.h>
15+
#include <limits.h>
16+
#include <sys/types.h>
17+
#include <sys/stat.h>
18+
#include <fcntl.h>
19+
#include <unistd.h>
20+
#include <sys/mman.h>
21+
#include <ccan/array_size.h>
22+
23+
#include <util/compiler.h>
24+
#include <util/mmio.h>
25+
#include <rdma/ib_user_ioctl_cmds.h>
26+
#include <rdma/xsc_user_ioctl_cmds.h>
27+
#include <infiniband/cmd_write.h>
28+
29+
#include "xscale.h"
30+
#include "xsc-abi.h"
31+
32+
int xsc_query_port(struct ibv_context *context, u8 port,
33+
struct ibv_port_attr *attr)
34+
{
35+
struct ibv_query_port cmd;
36+
37+
return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
38+
}
39+
40+
static void xsc_set_fw_version(struct ibv_device_attr *attr,
41+
union xsc_ib_fw_ver *fw_ver)
42+
{
43+
u8 ver_major = fw_ver->s.ver_major;
44+
u8 ver_minor = fw_ver->s.ver_minor;
45+
u16 ver_patch = fw_ver->s.ver_patch;
46+
u32 ver_tweak = fw_ver->s.ver_tweak;
47+
48+
if (ver_tweak == 0) {
49+
snprintf(attr->fw_ver, sizeof(attr->fw_ver), "v%u.%u.%u",
50+
ver_major, ver_minor, ver_patch);
51+
} else {
52+
snprintf(attr->fw_ver, sizeof(attr->fw_ver), "v%u.%u.%u+%u",
53+
ver_major, ver_minor, ver_patch, ver_tweak);
54+
}
55+
}
56+
57+
int xsc_query_device_ex(struct ibv_context *context,
58+
const struct ibv_query_device_ex_input *input,
59+
struct ibv_device_attr_ex *attr, size_t attr_size)
60+
{
61+
struct xsc_context *xctx = to_xctx(context);
62+
struct xsc_query_device_ex_resp resp = {};
63+
size_t resp_size =
64+
(xctx->cmds_supp_uhw & XSC_USER_CMDS_SUPP_UHW_QUERY_DEVICE) ?
65+
sizeof(resp) :
66+
sizeof(resp.ibv_resp);
67+
struct ibv_device_attr *a;
68+
union xsc_ib_fw_ver raw_fw_ver;
69+
int err;
70+
71+
raw_fw_ver.data = 0;
72+
err = ibv_cmd_query_device_any(context, input, attr, attr_size,
73+
&resp.ibv_resp, &resp_size);
74+
if (err)
75+
return err;
76+
77+
if (attr_size >= offsetofend(struct ibv_device_attr_ex, tso_caps)) {
78+
attr->tso_caps.max_tso = resp.tso_caps.max_tso;
79+
attr->tso_caps.supported_qpts = resp.tso_caps.supported_qpts;
80+
}
81+
if (attr_size >= offsetofend(struct ibv_device_attr_ex, rss_caps)) {
82+
attr->rss_caps.rx_hash_fields_mask =
83+
resp.rss_caps.rx_hash_fields_mask;
84+
attr->rss_caps.rx_hash_function =
85+
resp.rss_caps.rx_hash_function;
86+
}
87+
if (attr_size >=
88+
offsetofend(struct ibv_device_attr_ex, packet_pacing_caps)) {
89+
attr->packet_pacing_caps.qp_rate_limit_min =
90+
resp.packet_pacing_caps.qp_rate_limit_min;
91+
attr->packet_pacing_caps.qp_rate_limit_max =
92+
resp.packet_pacing_caps.qp_rate_limit_max;
93+
attr->packet_pacing_caps.supported_qpts =
94+
resp.packet_pacing_caps.supported_qpts;
95+
}
96+
97+
if (resp.xsc_ib_support_multi_pkt_send_wqes & XSC_IB_ALLOW_MPW)
98+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_MPW_ALLOWED;
99+
100+
if (resp.xsc_ib_support_multi_pkt_send_wqes & XSC_IB_SUPPORT_EMPW)
101+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_ENHANCED_MPW;
102+
103+
xctx->tunnel_offloads_caps = resp.tunnel_offloads_caps;
104+
xctx->packet_pacing_caps = resp.packet_pacing_caps;
105+
106+
if (resp.flags & XSC_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP)
107+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_CQE_128B_COMP;
108+
109+
if (resp.flags & XSC_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD)
110+
xctx->vendor_cap_flags |= XSC_VENDOR_CAP_FLAGS_CQE_128B_PAD;
111+
112+
raw_fw_ver.data = resp.ibv_resp.base.fw_ver;
113+
a = &attr->orig_attr;
114+
xsc_set_fw_version(a, &raw_fw_ver);
115+
116+
return 0;
117+
}

‎providers/xscale/xsc-abi.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* Copyright (c) 2021 - 2022, Shanghai Yunsilicon Technology Co., Ltd.
4+
* All rights reserved.
5+
*/
6+
7+
#ifndef XSC_ABI_H
8+
#define XSC_ABI_H
9+
10+
#include <infiniband/kern-abi.h>
11+
#include <infiniband/verbs.h>
12+
#include <rdma/xsc-abi.h>
13+
#include <kernel-abi/xsc-abi.h>
14+
15+
#define XSC_UVERBS_MIN_ABI_VERSION 1
16+
#define XSC_UVERBS_MAX_ABI_VERSION 1
17+
18+
DECLARE_DRV_CMD(xsc_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
19+
xsc_ib_alloc_ucontext_req, xsc_ib_alloc_ucontext_resp);
20+
DECLARE_DRV_CMD(xsc_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty,
21+
xsc_ib_alloc_pd_resp);
22+
DECLARE_DRV_CMD(xsc_create_cq, IB_USER_VERBS_CMD_CREATE_CQ, xsc_ib_create_cq,
23+
xsc_ib_create_cq_resp);
24+
DECLARE_DRV_CMD(xsc_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ,
25+
xsc_ib_create_cq, xsc_ib_create_cq_resp);
26+
DECLARE_DRV_CMD(xsc_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
27+
xsc_ib_create_qp, xsc_ib_create_qp_resp);
28+
DECLARE_DRV_CMD(xsc_create_qp, IB_USER_VERBS_CMD_CREATE_QP, xsc_ib_create_qp,
29+
xsc_ib_create_qp_resp);
30+
DECLARE_DRV_CMD(xsc_query_device_ex, IB_USER_VERBS_EX_CMD_QUERY_DEVICE, empty,
31+
xsc_ib_query_device_resp);
32+
DECLARE_DRV_CMD(xsc_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP, empty,
33+
xsc_ib_modify_qp_resp);
34+
35+
struct xsc_modify_qp {
36+
struct ibv_modify_qp_ex ibv_cmd;
37+
__u32 comp_mask;
38+
struct xsc_ib_burst_info burst_info;
39+
__u32 reserved;
40+
};
41+
42+
#endif /* XSC_ABI_H */

0 commit comments

Comments
 (0)