Skip to content

Commit 9167dbd

Browse files
author
Maxim Levitsky
committed
RDMA/mana_ib: add support of multiple ports
JIRA: https://issues.redhat.com/browse/RHEL-109583 commit 60c9a34 Author: Konstantin Taranov <kotaranov@microsoft.com> Date: Tue Jul 22 01:55:15 2025 -0700 RDMA/mana_ib: add support of multiple ports If the HW indicates support of multiple ports for rdma, create an IB device with a port per netdev in the ethernet mana driver. CM is only available on port 1, but RC QPs are supported on all ports. Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com> Link: https://patch.msgid.link/1753174515-23634-1-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky <leon@kernel.org> Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>
1 parent 57eb515 commit 9167dbd

File tree

3 files changed

+71
-57
lines changed

3 files changed

+71
-57
lines changed

‎drivers/infiniband/hw/mana/device.c‎

Lines changed: 60 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -78,28 +78,31 @@ static int mana_ib_netdev_event(struct notifier_block *this,
7878
struct gdma_context *gc = dev->gdma_dev->gdma_context;
7979
struct mana_context *mc = gc->mana.driver_data;
8080
struct net_device *ndev;
81+
int i;
8182

8283
/* Only process events from our parent device */
83-
if (event_dev != mc->ports[0])
84-
return NOTIFY_DONE;
85-
86-
switch (event) {
87-
case NETDEV_CHANGEUPPER:
88-
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
89-
/*
90-
* RDMA core will setup GID based on updated netdev.
91-
* It's not possible to race with the core as rtnl lock is being
92-
* held.
93-
*/
94-
ib_device_set_netdev(&dev->ib_dev, ndev, 1);
95-
96-
/* mana_get_primary_netdev() returns ndev with refcount held */
97-
netdev_put(ndev, &dev->dev_tracker);
98-
99-
return NOTIFY_OK;
100-
default:
101-
return NOTIFY_DONE;
102-
}
84+
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
85+
if (event_dev == mc->ports[i]) {
86+
switch (event) {
87+
case NETDEV_CHANGEUPPER:
88+
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
89+
/*
90+
* RDMA core will setup GID based on updated netdev.
91+
* It's not possible to race with the core as rtnl lock is being
92+
* held.
93+
*/
94+
ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
95+
96+
/* mana_get_primary_netdev() returns ndev with refcount held */
97+
if (ndev)
98+
netdev_put(ndev, &dev->dev_tracker);
99+
100+
return NOTIFY_OK;
101+
default:
102+
return NOTIFY_DONE;
103+
}
104+
}
105+
return NOTIFY_DONE;
103106
}
104107

105108
static int mana_ib_probe(struct auxiliary_device *adev,
@@ -112,7 +115,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
112115
struct net_device *ndev;
113116
struct mana_ib_dev *dev;
114117
u8 mac_addr[ETH_ALEN];
115-
int ret;
118+
int ret, i;
116119

117120
dev = ib_alloc_device(mana_ib_dev, ib_dev);
118121
if (!dev)
@@ -127,34 +130,11 @@ static int mana_ib_probe(struct auxiliary_device *adev,
127130

128131
if (mana_ib_is_rnic(dev)) {
129132
dev->ib_dev.phys_port_cnt = 1;
130-
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
131-
if (!ndev) {
132-
ret = -ENODEV;
133-
ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");
134-
goto free_ib_device;
135-
}
136-
ether_addr_copy(mac_addr, ndev->dev_addr);
137-
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);
138-
ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);
139-
/* mana_get_primary_netdev() returns ndev with refcount held */
140-
netdev_put(ndev, &dev->dev_tracker);
141-
if (ret) {
142-
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
143-
goto free_ib_device;
144-
}
145-
146-
dev->nb.notifier_call = mana_ib_netdev_event;
147-
ret = register_netdevice_notifier(&dev->nb);
148-
if (ret) {
149-
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",
150-
ret);
151-
goto free_ib_device;
152-
}
153-
133+
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
154134
ret = mana_ib_gd_query_adapter_caps(dev);
155135
if (ret) {
156136
ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
157-
goto deregister_net_notifier;
137+
goto free_ib_device;
158138
}
159139

160140
ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
@@ -164,16 +144,42 @@ static int mana_ib_probe(struct auxiliary_device *adev,
164144
ret = mana_ib_create_eqs(dev);
165145
if (ret) {
166146
ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
167-
goto deregister_net_notifier;
147+
goto free_ib_device;
168148
}
169149

170150
ret = mana_ib_gd_create_rnic_adapter(dev);
171151
if (ret)
172152
goto destroy_eqs;
173153

174-
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
154+
if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
155+
dev->ib_dev.phys_port_cnt = mc->num_ports;
156+
157+
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
158+
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
159+
if (!ndev) {
160+
ret = -ENODEV;
161+
ibdev_err(&dev->ib_dev,
162+
"Failed to get netdev for IB port %d", i + 1);
163+
goto destroy_rnic;
164+
}
165+
ether_addr_copy(mac_addr, ndev->dev_addr);
166+
ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
167+
/* mana_get_primary_netdev() returns ndev with refcount held */
168+
netdev_put(ndev, &dev->dev_tracker);
169+
if (ret) {
170+
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
171+
goto destroy_rnic;
172+
}
173+
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
174+
if (ret) {
175+
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
176+
goto destroy_rnic;
177+
}
178+
}
179+
dev->nb.notifier_call = mana_ib_netdev_event;
180+
ret = register_netdevice_notifier(&dev->nb);
175181
if (ret) {
176-
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
182+
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
177183
goto destroy_rnic;
178184
}
179185
} else {
@@ -189,7 +195,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
189195
MANA_AV_BUFFER_SIZE, 0);
190196
if (!dev->av_pool) {
191197
ret = -ENOMEM;
192-
goto destroy_rnic;
198+
goto deregister_net_notifier;
193199
}
194200

195201
ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@@ -206,15 +212,15 @@ static int mana_ib_probe(struct auxiliary_device *adev,
206212

207213
deallocate_pool:
208214
dma_pool_destroy(dev->av_pool);
215+
deregister_net_notifier:
216+
if (mana_ib_is_rnic(dev))
217+
unregister_netdevice_notifier(&dev->nb);
209218
destroy_rnic:
210219
if (mana_ib_is_rnic(dev))
211220
mana_ib_gd_destroy_rnic_adapter(dev);
212221
destroy_eqs:
213222
if (mana_ib_is_rnic(dev))
214223
mana_ib_destroy_eqs(dev);
215-
deregister_net_notifier:
216-
if (mana_ib_is_rnic(dev))
217-
unregister_netdevice_notifier(&dev->nb);
218224
free_ib_device:
219225
xa_destroy(&dev->qp_table_wq);
220226
ib_dealloc_device(&dev->ib_dev);
@@ -228,9 +234,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
228234
ib_unregister_device(&dev->ib_dev);
229235
dma_pool_destroy(dev->av_pool);
230236
if (mana_ib_is_rnic(dev)) {
237+
unregister_netdevice_notifier(&dev->nb);
231238
mana_ib_gd_destroy_rnic_adapter(dev);
232239
mana_ib_destroy_eqs(dev);
233-
unregister_netdevice_notifier(&dev->nb);
234240
}
235241
xa_destroy(&dev->qp_table_wq);
236242
ib_dealloc_device(&dev->ib_dev);

‎drivers/infiniband/hw/mana/main.c‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
563563
immutable->gid_tbl_len = attr.gid_tbl_len;
564564

565565
if (mana_ib_is_rnic(dev)) {
566-
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
567-
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
566+
if (port_num == 1) {
567+
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
568+
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
569+
} else {
570+
immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
571+
| RDMA_CORE_CAP_ETH_AH;
572+
immutable->max_mad_size = 0;
573+
}
568574
} else {
569575
immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
570576
}
@@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
633639
props->pkey_tbl_len = 1;
634640
if (mana_ib_is_rnic(dev)) {
635641
props->gid_tbl_len = 16;
636-
props->port_cap_flags = IB_PORT_CM_SUP;
637642
props->ip_gids = true;
643+
if (port == 1)
644+
props->port_cap_flags = IB_PORT_CM_SUP;
638645
}
639646

640647
return 0;

‎drivers/infiniband/hw/mana/mana_ib.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req {
220220
enum mana_ib_adapter_features {
221221
MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
222222
MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
223+
MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
223224
};
224225

225226
struct mana_ib_query_adapter_caps_resp {

0 commit comments

Comments
 (0)