Skip to content

Commit 9a43839

Browse files
ptalbertptalbert
committed
Merge: Backport more upstream MANA patches to RHEL 9.8
MR: https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9/-/merge_requests/7347 JIRA: https://issues.redhat.com/browse/RHEL-109583 Tested: passed basic tests on an Azure D2s_v6 instance. Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com> Approved-by: John W. Linville <linville@redhat.com> Approved-by: Michal Schmidt <mschmidt@redhat.com> Approved-by: CKI KWF Bot <cki-ci-bot+kwf-gitlab-com@redhat.com> Merged-by: Patrick Talbert <ptalbert@redhat.com>
2 parents f5efc6e + a6126d1 commit 9a43839

File tree

17 files changed

+1048
-185
lines changed

17 files changed

+1048
-185
lines changed

‎drivers/infiniband/hw/mana/counters.c‎

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,32 @@ static const struct rdma_stat_desc mana_ib_port_stats_desc[] = {
3232
[MANA_IB_RATE_INC_EVENTS].name = "rate_inc_events",
3333
[MANA_IB_NUM_QPS_RECOVERED].name = "num_qps_recovered",
3434
[MANA_IB_CURRENT_RATE].name = "current_rate",
35+
[MANA_IB_DUP_RX_REQ].name = "dup_rx_requests",
36+
[MANA_IB_TX_BYTES].name = "tx_bytes",
37+
[MANA_IB_RX_BYTES].name = "rx_bytes",
38+
[MANA_IB_RX_SEND_REQ].name = "rx_send_requests",
39+
[MANA_IB_RX_WRITE_REQ].name = "rx_write_requests",
40+
[MANA_IB_RX_READ_REQ].name = "rx_read_requests",
41+
[MANA_IB_TX_PKT].name = "tx_packets",
42+
[MANA_IB_RX_PKT].name = "rx_packets",
3543
};
3644

45+
static const struct rdma_stat_desc mana_ib_device_stats_desc[] = {
46+
[MANA_IB_SENT_CNPS].name = "sent_cnps",
47+
[MANA_IB_RECEIVED_ECNS].name = "received_ecns",
48+
[MANA_IB_RECEIVED_CNP_COUNT].name = "received_cnp_count",
49+
[MANA_IB_QP_CONGESTED_EVENTS].name = "qp_congested_events",
50+
[MANA_IB_QP_RECOVERED_EVENTS].name = "qp_recovered_events",
51+
[MANA_IB_DEV_RATE_INC_EVENTS].name = "rate_inc_events",
52+
};
53+
54+
struct rdma_hw_stats *mana_ib_alloc_hw_device_stats(struct ib_device *ibdev)
55+
{
56+
return rdma_alloc_hw_stats_struct(mana_ib_device_stats_desc,
57+
ARRAY_SIZE(mana_ib_device_stats_desc),
58+
RDMA_HW_STATS_DEFAULT_LIFESPAN);
59+
}
60+
3761
struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
3862
u32 port_num)
3963
{
@@ -42,8 +66,39 @@ struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
4266
RDMA_HW_STATS_DEFAULT_LIFESPAN);
4367
}
4468

45-
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
46-
u32 port_num, int index)
69+
static int mana_ib_get_hw_device_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats)
70+
{
71+
struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,
72+
ib_dev);
73+
struct mana_rnic_query_device_cntrs_resp resp = {};
74+
struct mana_rnic_query_device_cntrs_req req = {};
75+
int err;
76+
77+
mana_gd_init_req_hdr(&req.hdr, MANA_IB_QUERY_DEVICE_COUNTERS,
78+
sizeof(req), sizeof(resp));
79+
req.hdr.dev_id = mdev->gdma_dev->dev_id;
80+
req.adapter = mdev->adapter_handle;
81+
82+
err = mana_gd_send_request(mdev_to_gc(mdev), sizeof(req), &req,
83+
sizeof(resp), &resp);
84+
if (err) {
85+
ibdev_err(&mdev->ib_dev, "Failed to query device counters err %d",
86+
err);
87+
return err;
88+
}
89+
90+
stats->value[MANA_IB_SENT_CNPS] = resp.sent_cnps;
91+
stats->value[MANA_IB_RECEIVED_ECNS] = resp.received_ecns;
92+
stats->value[MANA_IB_RECEIVED_CNP_COUNT] = resp.received_cnp_count;
93+
stats->value[MANA_IB_QP_CONGESTED_EVENTS] = resp.qp_congested_events;
94+
stats->value[MANA_IB_QP_RECOVERED_EVENTS] = resp.qp_recovered_events;
95+
stats->value[MANA_IB_DEV_RATE_INC_EVENTS] = resp.rate_inc_events;
96+
97+
return ARRAY_SIZE(mana_ib_device_stats_desc);
98+
}
99+
100+
static int mana_ib_get_hw_port_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
101+
u32 port_num)
47102
{
48103
struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,
49104
ib_dev);
@@ -53,6 +108,7 @@ int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
53108

54109
mana_gd_init_req_hdr(&req.hdr, MANA_IB_QUERY_VF_COUNTERS,
55110
sizeof(req), sizeof(resp));
111+
req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
56112
req.hdr.dev_id = mdev->gdma_dev->dev_id;
57113
req.adapter = mdev->adapter_handle;
58114

@@ -101,5 +157,23 @@ int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
101157
stats->value[MANA_IB_NUM_QPS_RECOVERED] = resp.num_qps_recovered;
102158
stats->value[MANA_IB_CURRENT_RATE] = resp.current_rate;
103159

160+
stats->value[MANA_IB_DUP_RX_REQ] = resp.dup_rx_req;
161+
stats->value[MANA_IB_TX_BYTES] = resp.tx_bytes;
162+
stats->value[MANA_IB_RX_BYTES] = resp.rx_bytes;
163+
stats->value[MANA_IB_RX_SEND_REQ] = resp.rx_send_req;
164+
stats->value[MANA_IB_RX_WRITE_REQ] = resp.rx_write_req;
165+
stats->value[MANA_IB_RX_READ_REQ] = resp.rx_read_req;
166+
stats->value[MANA_IB_TX_PKT] = resp.tx_pkt;
167+
stats->value[MANA_IB_RX_PKT] = resp.rx_pkt;
168+
104169
return ARRAY_SIZE(mana_ib_port_stats_desc);
105170
}
171+
172+
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
173+
u32 port_num, int index)
174+
{
175+
if (!port_num)
176+
return mana_ib_get_hw_device_stats(ibdev, stats);
177+
else
178+
return mana_ib_get_hw_port_stats(ibdev, stats, port_num);
179+
}

‎drivers/infiniband/hw/mana/counters.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,28 @@ enum mana_ib_port_counters {
3535
MANA_IB_RATE_INC_EVENTS,
3636
MANA_IB_NUM_QPS_RECOVERED,
3737
MANA_IB_CURRENT_RATE,
38+
MANA_IB_DUP_RX_REQ,
39+
MANA_IB_TX_BYTES,
40+
MANA_IB_RX_BYTES,
41+
MANA_IB_RX_SEND_REQ,
42+
MANA_IB_RX_WRITE_REQ,
43+
MANA_IB_RX_READ_REQ,
44+
MANA_IB_TX_PKT,
45+
MANA_IB_RX_PKT,
46+
};
47+
48+
enum mana_ib_device_counters {
49+
MANA_IB_SENT_CNPS,
50+
MANA_IB_RECEIVED_ECNS,
51+
MANA_IB_RECEIVED_CNP_COUNT,
52+
MANA_IB_QP_CONGESTED_EVENTS,
53+
MANA_IB_QP_RECOVERED_EVENTS,
54+
MANA_IB_DEV_RATE_INC_EVENTS,
3855
};
3956

4057
struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
4158
u32 port_num);
59+
struct rdma_hw_stats *mana_ib_alloc_hw_device_stats(struct ib_device *ibdev);
4260
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
4361
u32 port_num, int index);
4462
#endif /* _COUNTERS_H_ */

‎drivers/infiniband/hw/mana/device.c‎

Lines changed: 66 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ static const struct ib_device_ops mana_ib_stats_ops = {
6666
.get_hw_stats = mana_ib_get_hw_stats,
6767
};
6868

69+
static const struct ib_device_ops mana_ib_device_stats_ops = {
70+
.alloc_hw_device_stats = mana_ib_alloc_hw_device_stats,
71+
};
72+
6973
static int mana_ib_netdev_event(struct notifier_block *this,
7074
unsigned long event, void *ptr)
7175
{
@@ -74,28 +78,31 @@ static int mana_ib_netdev_event(struct notifier_block *this,
7478
struct gdma_context *gc = dev->gdma_dev->gdma_context;
7579
struct mana_context *mc = gc->mana.driver_data;
7680
struct net_device *ndev;
81+
int i;
7782

7883
/* Only process events from our parent device */
79-
if (event_dev != mc->ports[0])
80-
return NOTIFY_DONE;
81-
82-
switch (event) {
83-
case NETDEV_CHANGEUPPER:
84-
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
85-
/*
86-
* RDMA core will setup GID based on updated netdev.
87-
* It's not possible to race with the core as rtnl lock is being
88-
* held.
89-
*/
90-
ib_device_set_netdev(&dev->ib_dev, ndev, 1);
91-
92-
/* mana_get_primary_netdev() returns ndev with refcount held */
93-
netdev_put(ndev, &dev->dev_tracker);
94-
95-
return NOTIFY_OK;
96-
default:
97-
return NOTIFY_DONE;
98-
}
84+
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
85+
if (event_dev == mc->ports[i]) {
86+
switch (event) {
87+
case NETDEV_CHANGEUPPER:
88+
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
89+
/*
90+
* RDMA core will setup GID based on updated netdev.
91+
* It's not possible to race with the core as rtnl lock is being
92+
* held.
93+
*/
94+
ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
95+
96+
/* mana_get_primary_netdev() returns ndev with refcount held */
97+
if (ndev)
98+
netdev_put(ndev, &dev->dev_tracker);
99+
100+
return NOTIFY_OK;
101+
default:
102+
return NOTIFY_DONE;
103+
}
104+
}
105+
return NOTIFY_DONE;
99106
}
100107

101108
static int mana_ib_probe(struct auxiliary_device *adev,
@@ -108,7 +115,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
108115
struct net_device *ndev;
109116
struct mana_ib_dev *dev;
110117
u8 mac_addr[ETH_ALEN];
111-
int ret;
118+
int ret, i;
112119

113120
dev = ib_alloc_device(mana_ib_dev, ib_dev);
114121
if (!dev)
@@ -123,51 +130,56 @@ static int mana_ib_probe(struct auxiliary_device *adev,
123130

124131
if (mana_ib_is_rnic(dev)) {
125132
dev->ib_dev.phys_port_cnt = 1;
126-
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
127-
if (!ndev) {
128-
ret = -ENODEV;
129-
ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");
130-
goto free_ib_device;
131-
}
132-
ether_addr_copy(mac_addr, ndev->dev_addr);
133-
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);
134-
ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);
135-
/* mana_get_primary_netdev() returns ndev with refcount held */
136-
netdev_put(ndev, &dev->dev_tracker);
137-
if (ret) {
138-
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
139-
goto free_ib_device;
140-
}
141-
142-
dev->nb.notifier_call = mana_ib_netdev_event;
143-
ret = register_netdevice_notifier(&dev->nb);
144-
if (ret) {
145-
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",
146-
ret);
147-
goto free_ib_device;
148-
}
149-
133+
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
150134
ret = mana_ib_gd_query_adapter_caps(dev);
151135
if (ret) {
152136
ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
153-
goto deregister_net_notifier;
137+
goto free_ib_device;
154138
}
155139

156140
ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
141+
if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT)
142+
ib_set_device_ops(&dev->ib_dev, &mana_ib_device_stats_ops);
157143

158144
ret = mana_ib_create_eqs(dev);
159145
if (ret) {
160146
ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
161-
goto deregister_net_notifier;
147+
goto free_ib_device;
162148
}
163149

164150
ret = mana_ib_gd_create_rnic_adapter(dev);
165151
if (ret)
166152
goto destroy_eqs;
167153

168-
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
154+
if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
155+
dev->ib_dev.phys_port_cnt = mc->num_ports;
156+
157+
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
158+
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
159+
if (!ndev) {
160+
ret = -ENODEV;
161+
ibdev_err(&dev->ib_dev,
162+
"Failed to get netdev for IB port %d", i + 1);
163+
goto destroy_rnic;
164+
}
165+
ether_addr_copy(mac_addr, ndev->dev_addr);
166+
ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
167+
/* mana_get_primary_netdev() returns ndev with refcount held */
168+
netdev_put(ndev, &dev->dev_tracker);
169+
if (ret) {
170+
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
171+
goto destroy_rnic;
172+
}
173+
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
174+
if (ret) {
175+
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
176+
goto destroy_rnic;
177+
}
178+
}
179+
dev->nb.notifier_call = mana_ib_netdev_event;
180+
ret = register_netdevice_notifier(&dev->nb);
169181
if (ret) {
170-
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
182+
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
171183
goto destroy_rnic;
172184
}
173185
} else {
@@ -183,7 +195,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
183195
MANA_AV_BUFFER_SIZE, 0);
184196
if (!dev->av_pool) {
185197
ret = -ENOMEM;
186-
goto destroy_rnic;
198+
goto deregister_net_notifier;
187199
}
188200

189201
ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@@ -200,15 +212,15 @@ static int mana_ib_probe(struct auxiliary_device *adev,
200212

201213
deallocate_pool:
202214
dma_pool_destroy(dev->av_pool);
215+
deregister_net_notifier:
216+
if (mana_ib_is_rnic(dev))
217+
unregister_netdevice_notifier(&dev->nb);
203218
destroy_rnic:
204219
if (mana_ib_is_rnic(dev))
205220
mana_ib_gd_destroy_rnic_adapter(dev);
206221
destroy_eqs:
207222
if (mana_ib_is_rnic(dev))
208223
mana_ib_destroy_eqs(dev);
209-
deregister_net_notifier:
210-
if (mana_ib_is_rnic(dev))
211-
unregister_netdevice_notifier(&dev->nb);
212224
free_ib_device:
213225
xa_destroy(&dev->qp_table_wq);
214226
ib_dealloc_device(&dev->ib_dev);
@@ -222,9 +234,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
222234
ib_unregister_device(&dev->ib_dev);
223235
dma_pool_destroy(dev->av_pool);
224236
if (mana_ib_is_rnic(dev)) {
237+
unregister_netdevice_notifier(&dev->nb);
225238
mana_ib_gd_destroy_rnic_adapter(dev);
226239
mana_ib_destroy_eqs(dev);
227-
unregister_netdevice_notifier(&dev->nb);
228240
}
229241
xa_destroy(&dev->qp_table_wq);
230242
ib_dealloc_device(&dev->ib_dev);

‎drivers/infiniband/hw/mana/main.c‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
563563
immutable->gid_tbl_len = attr.gid_tbl_len;
564564

565565
if (mana_ib_is_rnic(dev)) {
566-
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
567-
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
566+
if (port_num == 1) {
567+
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
568+
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
569+
} else {
570+
immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
571+
| RDMA_CORE_CAP_ETH_AH;
572+
immutable->max_mad_size = 0;
573+
}
568574
} else {
569575
immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
570576
}
@@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
633639
props->pkey_tbl_len = 1;
634640
if (mana_ib_is_rnic(dev)) {
635641
props->gid_tbl_len = 16;
636-
props->port_cap_flags = IB_PORT_CM_SUP;
637642
props->ip_gids = true;
643+
if (port == 1)
644+
props->port_cap_flags = IB_PORT_CM_SUP;
638645
}
639646

640647
return 0;

0 commit comments

Comments
 (0)