Skip to content

Commit edb5969

Browse files
author
Maxim Levitsky
committed
RDMA/mana_ib: add support of multiple ports
JIRA: https://issues.redhat.com/browse/RHEL-109580 commit 60c9a34 Author: Konstantin Taranov <kotaranov@microsoft.com> Date: Tue Jul 22 01:55:15 2025 -0700 RDMA/mana_ib: add support of multiple ports If the HW indicates support of multiple ports for rdma, create an IB device with a port per netdev in the ethernet mana driver. CM is only available on port 1, but RC QPs are supported on all ports. Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com> Link: https://patch.msgid.link/1753174515-23634-1-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky <leon@kernel.org> Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>
1 parent ce45fa8 commit edb5969

File tree

3 files changed

+71
-57
lines changed

3 files changed

+71
-57
lines changed

‎drivers/infiniband/hw/mana/device.c‎

Lines changed: 60 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -77,28 +77,31 @@ static int mana_ib_netdev_event(struct notifier_block *this,
7777
struct gdma_context *gc = dev->gdma_dev->gdma_context;
7878
struct mana_context *mc = gc->mana.driver_data;
7979
struct net_device *ndev;
80+
int i;
8081

8182
/* Only process events from our parent device */
82-
if (event_dev != mc->ports[0])
83-
return NOTIFY_DONE;
84-
85-
switch (event) {
86-
case NETDEV_CHANGEUPPER:
87-
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
88-
/*
89-
* RDMA core will setup GID based on updated netdev.
90-
* It's not possible to race with the core as rtnl lock is being
91-
* held.
92-
*/
93-
ib_device_set_netdev(&dev->ib_dev, ndev, 1);
94-
95-
/* mana_get_primary_netdev() returns ndev with refcount held */
96-
netdev_put(ndev, &dev->dev_tracker);
97-
98-
return NOTIFY_OK;
99-
default:
100-
return NOTIFY_DONE;
101-
}
83+
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
84+
if (event_dev == mc->ports[i]) {
85+
switch (event) {
86+
case NETDEV_CHANGEUPPER:
87+
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
88+
/*
89+
* RDMA core will setup GID based on updated netdev.
90+
* It's not possible to race with the core as rtnl lock is being
91+
* held.
92+
*/
93+
ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
94+
95+
/* mana_get_primary_netdev() returns ndev with refcount held */
96+
if (ndev)
97+
netdev_put(ndev, &dev->dev_tracker);
98+
99+
return NOTIFY_OK;
100+
default:
101+
return NOTIFY_DONE;
102+
}
103+
}
104+
return NOTIFY_DONE;
102105
}
103106

104107
static int mana_ib_probe(struct auxiliary_device *adev,
@@ -111,7 +114,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
111114
struct net_device *ndev;
112115
struct mana_ib_dev *dev;
113116
u8 mac_addr[ETH_ALEN];
114-
int ret;
117+
int ret, i;
115118

116119
dev = ib_alloc_device(mana_ib_dev, ib_dev);
117120
if (!dev)
@@ -126,34 +129,11 @@ static int mana_ib_probe(struct auxiliary_device *adev,
126129

127130
if (mana_ib_is_rnic(dev)) {
128131
dev->ib_dev.phys_port_cnt = 1;
129-
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
130-
if (!ndev) {
131-
ret = -ENODEV;
132-
ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");
133-
goto free_ib_device;
134-
}
135-
ether_addr_copy(mac_addr, ndev->dev_addr);
136-
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);
137-
ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);
138-
/* mana_get_primary_netdev() returns ndev with refcount held */
139-
netdev_put(ndev, &dev->dev_tracker);
140-
if (ret) {
141-
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
142-
goto free_ib_device;
143-
}
144-
145-
dev->nb.notifier_call = mana_ib_netdev_event;
146-
ret = register_netdevice_notifier(&dev->nb);
147-
if (ret) {
148-
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",
149-
ret);
150-
goto free_ib_device;
151-
}
152-
132+
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
153133
ret = mana_ib_gd_query_adapter_caps(dev);
154134
if (ret) {
155135
ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
156-
goto deregister_net_notifier;
136+
goto free_ib_device;
157137
}
158138

159139
ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
@@ -163,16 +143,42 @@ static int mana_ib_probe(struct auxiliary_device *adev,
163143
ret = mana_ib_create_eqs(dev);
164144
if (ret) {
165145
ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
166-
goto deregister_net_notifier;
146+
goto free_ib_device;
167147
}
168148

169149
ret = mana_ib_gd_create_rnic_adapter(dev);
170150
if (ret)
171151
goto destroy_eqs;
172152

173-
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
153+
if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
154+
dev->ib_dev.phys_port_cnt = mc->num_ports;
155+
156+
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
157+
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
158+
if (!ndev) {
159+
ret = -ENODEV;
160+
ibdev_err(&dev->ib_dev,
161+
"Failed to get netdev for IB port %d", i + 1);
162+
goto destroy_rnic;
163+
}
164+
ether_addr_copy(mac_addr, ndev->dev_addr);
165+
ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
166+
/* mana_get_primary_netdev() returns ndev with refcount held */
167+
netdev_put(ndev, &dev->dev_tracker);
168+
if (ret) {
169+
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
170+
goto destroy_rnic;
171+
}
172+
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
173+
if (ret) {
174+
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
175+
goto destroy_rnic;
176+
}
177+
}
178+
dev->nb.notifier_call = mana_ib_netdev_event;
179+
ret = register_netdevice_notifier(&dev->nb);
174180
if (ret) {
175-
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
181+
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
176182
goto destroy_rnic;
177183
}
178184
} else {
@@ -188,7 +194,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
188194
MANA_AV_BUFFER_SIZE, 0);
189195
if (!dev->av_pool) {
190196
ret = -ENOMEM;
191-
goto destroy_rnic;
197+
goto deregister_net_notifier;
192198
}
193199

194200
ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@@ -205,15 +211,15 @@ static int mana_ib_probe(struct auxiliary_device *adev,
205211

206212
deallocate_pool:
207213
dma_pool_destroy(dev->av_pool);
214+
deregister_net_notifier:
215+
if (mana_ib_is_rnic(dev))
216+
unregister_netdevice_notifier(&dev->nb);
208217
destroy_rnic:
209218
if (mana_ib_is_rnic(dev))
210219
mana_ib_gd_destroy_rnic_adapter(dev);
211220
destroy_eqs:
212221
if (mana_ib_is_rnic(dev))
213222
mana_ib_destroy_eqs(dev);
214-
deregister_net_notifier:
215-
if (mana_ib_is_rnic(dev))
216-
unregister_netdevice_notifier(&dev->nb);
217223
free_ib_device:
218224
xa_destroy(&dev->qp_table_wq);
219225
ib_dealloc_device(&dev->ib_dev);
@@ -227,9 +233,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
227233
ib_unregister_device(&dev->ib_dev);
228234
dma_pool_destroy(dev->av_pool);
229235
if (mana_ib_is_rnic(dev)) {
236+
unregister_netdevice_notifier(&dev->nb);
230237
mana_ib_gd_destroy_rnic_adapter(dev);
231238
mana_ib_destroy_eqs(dev);
232-
unregister_netdevice_notifier(&dev->nb);
233239
}
234240
xa_destroy(&dev->qp_table_wq);
235241
ib_dealloc_device(&dev->ib_dev);

‎drivers/infiniband/hw/mana/main.c‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
563563
immutable->gid_tbl_len = attr.gid_tbl_len;
564564

565565
if (mana_ib_is_rnic(dev)) {
566-
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
567-
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
566+
if (port_num == 1) {
567+
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
568+
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
569+
} else {
570+
immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
571+
| RDMA_CORE_CAP_ETH_AH;
572+
immutable->max_mad_size = 0;
573+
}
568574
} else {
569575
immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
570576
}
@@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
633639
props->pkey_tbl_len = 1;
634640
if (mana_ib_is_rnic(dev)) {
635641
props->gid_tbl_len = 16;
636-
props->port_cap_flags = IB_PORT_CM_SUP;
637642
props->ip_gids = true;
643+
if (port == 1)
644+
props->port_cap_flags = IB_PORT_CM_SUP;
638645
}
639646

640647
return 0;

‎drivers/infiniband/hw/mana/mana_ib.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req {
220220
enum mana_ib_adapter_features {
221221
MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
222222
MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
223+
MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
223224
};
224225

225226
struct mana_ib_query_adapter_caps_resp {

0 commit comments

Comments
 (0)