Skip to content

Commit a8deb6a

Browse files
Kalesh APsbasavapatna
Kalesh AP
authored and
sbasavapatna
committed
bnxt_re/lib: Direct Verbs: Support DBR and UMEM verbs
The following Direct Verb (DV) APIs have been implemented in this patch. Doorbell Region Direct Verbs: ----------------------------- - bnxt_re_dv_alloc_db_region(): This will allow the appliation to create extra doorbell regions and use the associated doorbell page index in dv_create_qp() and use the associated DB address while ringing the doorbell. - bnxt_re_dv_free_db_region(): Free the allocated doorbell region. - bnxt_re_dv_get_default_db_region(): Return the default doorbell page index and doorbell page address associated with the ucontext. Umem Registration Direct Verbs: ------------------------------- - bnxt_re_dv_umem_reg(): Register the user memory to be used by the application with the library. Application can register a large chunk of memory and use it during subsequent resource creation DV APIs. Note that the API terminates in the library and the app specified memory params (addr, len) are saved and a umem-handle is returned. That is, there is no ioctl to the driver at this point to map/pin the user memory. This memory is mapped/pinned later when the application creates the required resources (CQ/QP) using respective direct verbs. This is implemented in the next patch in this series. - bnxt_re_dv_umem_dereg(): Deregister the user memory specified by the umem-handle. Co-developed-by: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com> Signed-off-by: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com> Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com> Reviewed-by: Selvin Thyparampil Xavier <selvin.xavier@broadcom.com>
1 parent c3a5dc2 commit a8deb6a

File tree

5 files changed

+340
-0
lines changed

5 files changed

+340
-0
lines changed

‎providers/bnxt_re/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ rdma_provider(bnxt_re
33
main.c
44
memory.c
55
verbs.c
6+
dv.c
67
)

‎providers/bnxt_re/bnxt_re.map‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
global:
3+
openib_driver_init;
4+
bnxt_re_dv_alloc_db_region;
5+
bnxt_re_dv_free_db_region;
6+
bnxt_re_dv_umem_reg;
7+
bnxt_re_dv_umem_dereg;
8+
bnxt_re_dv_get_default_db_region;
9+
local: *;
10+
};

‎providers/bnxt_re/bnxt_re_dv.h‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright (c) 2025, Broadcom. All rights reserved. The term
3+
* Broadcom refers to Broadcom Limited and/or its subsidiaries.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or without
12+
* modification, are permitted provided that the following conditions
13+
* are met:
14+
*
15+
* 1. Redistributions of source code must retain the above copyright
16+
* notice, this list of conditions and the following disclaimer.
17+
* 2. Redistributions in binary form must reproduce the above copyright
18+
* notice, this list of conditions and the following disclaimer in
19+
* the documentation and/or other materials provided with the
20+
* distribution.
21+
*
22+
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
23+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24+
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25+
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
26+
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
29+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30+
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
32+
* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
*
34+
* Description: Direct verb support user interface header
35+
*/
36+
37+
#ifndef __BNXT_RE_DV_H__
38+
#define __BNXT_RE_DV_H__
39+
40+
#include <stdint.h>
41+
#include <infiniband/verbs.h>
42+
#ifdef __cplusplus
43+
extern "C" {
44+
#endif
45+
46+
struct bnxt_re_dv_db_region_attr {
47+
uint32_t handle;
48+
uint32_t dpi;
49+
uint64_t umdbr;
50+
__u64 *dbr;
51+
};
52+
53+
enum bnxt_re_dv_umem_in_flags {
54+
BNXT_RE_DV_UMEM_FLAGS_DMABUF = 1 << 0,
55+
};
56+
57+
struct bnxt_re_dv_umem_reg_attr {
58+
void *addr;
59+
size_t size;
60+
uint32_t access_flags;
61+
uint64_t pgsz_bitmap;
62+
uint64_t comp_mask;
63+
int dmabuf_fd;
64+
};
65+
66+
struct bnxt_re_dv_db_region_attr *
67+
bnxt_re_dv_alloc_db_region(struct ibv_context *ctx);
68+
int bnxt_re_dv_free_db_region(struct ibv_context *ctx,
69+
struct bnxt_re_dv_db_region_attr *attr);
70+
int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx,
71+
struct bnxt_re_dv_db_region_attr *out);
72+
void *bnxt_re_dv_umem_reg(struct ibv_context *ibvctx,
73+
struct bnxt_re_dv_umem_reg_attr *in);
74+
int bnxt_re_dv_umem_dereg(void *umem_handle);
75+
#ifdef __cplusplus
76+
}
77+
#endif
78+
#endif /* __BNXT_RE_DV_H__ */

‎providers/bnxt_re/dv.c‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
/*
2+
* Copyright (c) 2025, Broadcom. All rights reserved. The term
3+
* Broadcom refers to Broadcom Limited and/or its subsidiaries.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or without
12+
* modification, are permitted provided that the following conditions
13+
* are met:
14+
*
15+
* 1. Redistributions of source code must retain the above copyright
16+
* notice, this list of conditions and the following disclaimer.
17+
* 2. Redistributions in binary form must reproduce the above copyright
18+
* notice, this list of conditions and the following disclaimer in
19+
* the documentation and/or other materials provided with the
20+
* distribution.
21+
*
22+
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
23+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24+
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25+
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
26+
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
29+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30+
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
32+
* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
*
34+
* Description: Direct verbs API function definitions.
35+
*/
36+
37+
#include <stdio.h>
38+
#include <sys/mman.h>
39+
40+
#include "main.h"
41+
#include "bnxt_re-abi.h"
42+
#include "bnxt_re_dv.h"
43+
#include "./verbs.h"
44+
#include "dv_internal.h"
45+
46+
/* Returns details about the default Doorbell page for ucontext */
47+
int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx,
48+
struct bnxt_re_dv_db_region_attr *out)
49+
{
50+
struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx);
51+
struct bnxt_re_dv_db_region_attr attr = {};
52+
int ret;
53+
54+
DECLARE_COMMAND_BUFFER(cmd,
55+
BNXT_RE_OBJECT_DBR,
56+
BNXT_RE_METHOD_DBR_QUERY,
57+
1);
58+
59+
fill_attr_out_ptr(cmd, BNXT_RE_DV_QUERY_DBR_ATTR, &attr);
60+
61+
ret = execute_ioctl(ibvctx, cmd);
62+
if (ret) {
63+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n", __func__, ret);
64+
return ret;
65+
}
66+
out->dbr = cntx->udpi.dbpage;
67+
out->dpi = attr.dpi;
68+
out->umdbr = attr.umdbr;
69+
return 0;
70+
}
71+
72+
int bnxt_re_dv_free_db_region(struct ibv_context *ctx,
73+
struct bnxt_re_dv_db_region_attr *attr)
74+
{
75+
struct bnxt_re_dev *dev = to_bnxt_re_dev(ctx->device);
76+
int ret;
77+
78+
DECLARE_COMMAND_BUFFER(cmd,
79+
BNXT_RE_OBJECT_DBR,
80+
BNXT_RE_METHOD_DBR_FREE,
81+
1);
82+
83+
if (attr->dbr != MAP_FAILED)
84+
munmap(attr->dbr, dev->pg_size);
85+
86+
bnxt_trace_dv(NULL, DEV "%s: DV DBR: handle: 0x%x\n", __func__, attr->handle);
87+
fill_attr_in_obj(cmd, BNXT_RE_DV_FREE_DBR_HANDLE, attr->handle);
88+
89+
ret = execute_ioctl(ctx, cmd);
90+
if (ret) {
91+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n",
92+
__func__, ret);
93+
errno = ret;
94+
return ret;
95+
}
96+
97+
free(attr);
98+
return 0;
99+
}
100+
101+
struct bnxt_re_dv_db_region_attr *
102+
bnxt_re_dv_alloc_db_region(struct ibv_context *ctx)
103+
{
104+
struct bnxt_re_dev *dev = to_bnxt_re_dev(ctx->device);
105+
struct bnxt_re_dv_db_region_attr attr = {}, *out;
106+
struct ib_uverbs_attr *handle;
107+
uint64_t mmap_offset = 0;
108+
int ret;
109+
110+
DECLARE_COMMAND_BUFFER(cmd,
111+
BNXT_RE_OBJECT_DBR,
112+
BNXT_RE_METHOD_DBR_ALLOC,
113+
3);
114+
115+
out = calloc(1, sizeof(*out));
116+
if (!out) {
117+
errno = ENOMEM;
118+
return NULL;
119+
}
120+
121+
handle = fill_attr_out_obj(cmd, BNXT_RE_DV_ALLOC_DBR_HANDLE);
122+
fill_attr_out_ptr(cmd, BNXT_RE_DV_ALLOC_DBR_ATTR, &attr);
123+
fill_attr_out_ptr(cmd, BNXT_RE_DV_ALLOC_DBR_OFFSET, &mmap_offset);
124+
125+
ret = execute_ioctl(ctx, cmd);
126+
if (ret) {
127+
fprintf(stderr, "%s: execute_ioctl() failed: %d\n",
128+
__func__, ret);
129+
free(out);
130+
errno = ret;
131+
return NULL;
132+
}
133+
out->handle = read_attr_obj(BNXT_RE_DV_ALLOC_DBR_HANDLE, handle);
134+
out->dpi = attr.dpi;
135+
out->umdbr = attr.umdbr;
136+
137+
out->dbr = mmap(NULL, dev->pg_size, PROT_WRITE,
138+
MAP_SHARED, ctx->cmd_fd, mmap_offset);
139+
if (out->dbr == MAP_FAILED) {
140+
fprintf(stderr, DEV "%s: mmap failed\n", __func__);
141+
bnxt_re_dv_free_db_region(ctx, out);
142+
errno = ENOMEM;
143+
return NULL;
144+
}
145+
bnxt_trace_dv(NULL, "%s: DV DBR: handle: 0x%x\n", __func__, out->handle);
146+
147+
return out;
148+
}
149+
150+
void *bnxt_re_dv_umem_reg(struct ibv_context *ibvctx, struct bnxt_re_dv_umem_reg_attr *in)
151+
{
152+
struct bnxt_re_dv_umem_internal *umem;
153+
int ret;
154+
155+
ret = ibv_dontfork_range(in->addr, in->size);
156+
if (ret) {
157+
errno = ret;
158+
return NULL;
159+
}
160+
161+
if (in->comp_mask & BNXT_RE_DV_UMEM_FLAGS_DMABUF &&
162+
(in->dmabuf_fd == -1)) {
163+
fprintf(stderr, "%s: failed: EBADF\n", __func__);
164+
errno = EBADF;
165+
goto err;
166+
}
167+
168+
umem = calloc(1, sizeof(*umem));
169+
if (!umem) {
170+
errno = ENOMEM;
171+
goto err;
172+
}
173+
174+
umem->context = ibvctx;
175+
umem->addr = in->addr;
176+
umem->size = in->size;
177+
umem->access_flags = in->access_flags;
178+
umem->pgsz_bitmap = in->pgsz_bitmap;
179+
umem->dmabuf_fd = in->dmabuf_fd;
180+
181+
bnxt_trace_dv(NULL, "%s: umem: %" PRIuPTR " addr: %" PRIuPTR " size: %zu\n",
182+
__func__, (uintptr_t)umem, (uintptr_t)umem->addr, umem->size);
183+
return (void *)umem;
184+
185+
err:
186+
ibv_dofork_range(in->addr, in->size);
187+
return NULL;
188+
}
189+
190+
int bnxt_re_dv_umem_dereg(void *umem_handle)
191+
{
192+
struct bnxt_re_dv_umem_internal *umem = umem_handle;
193+
194+
bnxt_trace_dv(NULL, "%s: DV Umem Dereg: handle: %" PRIuPTR "\n",
195+
__func__, (uintptr_t)umem);
196+
ibv_dofork_range(umem->addr, umem->size);
197+
free(umem);
198+
return 0;
199+
}

‎providers/bnxt_re/dv_internal.h‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Copyright (c) 2025, Broadcom. All rights reserved. The term
3+
* Broadcom refers to Broadcom Limited and/or its subsidiaries.
4+
*
5+
* This software is available to you under a choice of one of two
6+
* licenses. You may choose to be licensed under the terms of the GNU
7+
* General Public License (GPL) Version 2, available from the file
8+
* COPYING in the main directory of this source tree, or the
9+
* BSD license below:
10+
*
11+
* Redistribution and use in source and binary forms, with or without
12+
* modification, are permitted provided that the following conditions
13+
* are met:
14+
*
15+
* 1. Redistributions of source code must retain the above copyright
16+
* notice, this list of conditions and the following disclaimer.
17+
* 2. Redistributions in binary form must reproduce the above copyright
18+
* notice, this list of conditions and the following disclaimer in
19+
* the documentation and/or other materials provided with the
20+
* distribution.
21+
*
22+
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
23+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24+
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25+
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
26+
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
29+
* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30+
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
32+
* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
*
34+
* Description: Direct verb support user interface header
35+
*/
36+
37+
#ifndef __BNXT_RE_DV_INTERNAL_H__
38+
#define __BNXT_RE_DV_INTERNAL_H__
39+
40+
#include <stdint.h>
41+
#include <infiniband/verbs.h>
42+
43+
struct bnxt_re_dv_umem_internal {
44+
struct ibv_context *context;
45+
void *addr;
46+
size_t size;
47+
uint32_t access_flags;
48+
uint64_t pgsz_bitmap;
49+
int dmabuf_fd;
50+
};
51+
52+
#endif /* __BNXT_RE_DV_INTERNAL_H__ */

0 commit comments

Comments
 (0)