ipv6: Fix soft lockups in fib6_select_path under high next hop churn

JIRA: https://issues.redhat.com/browse/RHEL-73281
Upstream Status: net.git commit d9ccb18

Conflicts: context conflicts due to missing a lot selftest updates.

commit d9ccb18
Author: Omid Ehtemam-Haghighi <omid.ehtemamhaghighi@menlosecurity.com>
Date:   Tue Nov 5 17:02:36 2024 -0800

    ipv6: Fix soft lockups in fib6_select_path under high next hop churn

    Soft lockups have been observed on a cluster of Linux-based edge routers
    located in a highly dynamic environment. Using the `bird` service, these
    routers continuously update BGP-advertised routes due to frequently
    changing nexthop destinations, while also managing significant IPv6
    traffic. The lockups occur during the traversal of the multipath
    circular linked-list in the `fib6_select_path` function, particularly
    while iterating through the siblings in the list. The issue typically
    arises when the nodes of the linked list are unexpectedly deleted
    concurrently on a different core—indicated by their 'next' and
    'previous' elements pointing back to the node itself and their reference
    count dropping to zero. This results in an infinite loop, leading to a
    soft lockup that triggers a system panic via the watchdog timer.

    Apply RCU primitives in the problematic code sections to resolve the
    issue. Where necessary, update the references to fib6_siblings to
    annotate or use the RCU APIs.

    Include a test script that reproduces the issue. The script
    periodically updates the routing table while generating a heavy load
    of outgoing IPv6 traffic through multiple iperf3 clients. It
    consistently induces infinite soft lockups within a couple of minutes.

    Kernel log:

     0 [ffffbd13003e8d30] machine_kexec at ffffffff8ceaf3eb
     1 [ffffbd13003e8d90] __crash_kexec at ffffffff8d0120e3
     2 [ffffbd13003e8e58] panic at ffffffff8cef65d4
     3 [ffffbd13003e8ed8] watchdog_timer_fn at ffffffff8d05cb03
     4 [ffffbd13003e8f08] __hrtimer_run_queues at ffffffff8cfec62f
     5 [ffffbd13003e8f70] hrtimer_interrupt at ffffffff8cfed756
     6 [ffffbd13003e8fd0] __sysvec_apic_timer_interrupt at ffffffff8cea01af
     7 [ffffbd13003e8ff0] sysvec_apic_timer_interrupt at ffffffff8df1b83d
    -- <IRQ stack> --
     8 [ffffbd13003d3708] asm_sysvec_apic_timer_interrupt at ffffffff8e000ecb
        [exception RIP: fib6_select_path+299]
        RIP: ffffffff8ddafe7b  RSP: ffffbd13003d37b8  RFLAGS: 00000287
        RAX: ffff975850b43600  RBX: ffff975850b40200  RCX: 0000000000000000
        RDX: 000000003fffffff  RSI: 0000000051d383e4  RDI: ffff975850b43618
        RBP: ffffbd13003d3800   R8: 0000000000000000   R9: ffff975850b40200
        R10: 0000000000000000  R11: 0000000000000000  R12: ffffbd13003d3830
        R13: ffff975850b436a8  R14: ffff975850b43600  R15: 0000000000000007
        ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
     9 [ffffbd13003d3808] ip6_pol_route at ffffffff8ddb030c
    10 [ffffbd13003d3888] ip6_pol_route_input at ffffffff8ddb068c
    11 [ffffbd13003d3898] fib6_rule_lookup at ffffffff8ddf02b5
    12 [ffffbd13003d3928] ip6_route_input at ffffffff8ddb0f47
    13 [ffffbd13003d3a18] ip6_rcv_finish_core.constprop.0 at ffffffff8dd950d0
    14 [ffffbd13003d3a30] ip6_list_rcv_finish.constprop.0 at ffffffff8dd96274
    15 [ffffbd13003d3a98] ip6_sublist_rcv at ffffffff8dd96474
    16 [ffffbd13003d3af8] ipv6_list_rcv at ffffffff8dd96615
    17 [ffffbd13003d3b60] __netif_receive_skb_list_core at ffffffff8dc16fec
    18 [ffffbd13003d3be0] netif_receive_skb_list_internal at ffffffff8dc176b3
    19 [ffffbd13003d3c50] napi_gro_receive at ffffffff8dc565b9
    20 [ffffbd13003d3c80] ice_receive_skb at ffffffffc087e4f5 [ice]
    21 [ffffbd13003d3c90] ice_clean_rx_irq at ffffffffc0881b80 [ice]
    22 [ffffbd13003d3d20] ice_napi_poll at ffffffffc088232f [ice]
    23 [ffffbd13003d3d80] __napi_poll at ffffffff8dc18000
    24 [ffffbd13003d3db8] net_rx_action at ffffffff8dc18581
    25 [ffffbd13003d3e40] __do_softirq at ffffffff8df352e9
    26 [ffffbd13003d3eb0] run_ksoftirqd at ffffffff8ceffe47
    27 [ffffbd13003d3ec0] smpboot_thread_fn at ffffffff8cf36a30
    28 [ffffbd13003d3ee8] kthread at ffffffff8cf2b39f
    29 [ffffbd13003d3f28] ret_from_fork at ffffffff8ce5fa64
    30 [ffffbd13003d3f50] ret_from_fork_asm at ffffffff8ce03cbb

    Fixes: 66f5d6c ("ipv6: replace rwlock with rcu and spinlock in fib6_table")
    Reported-by: Adrian Oliver <kernel@aoliver.ca>
    Signed-off-by: Omid Ehtemam-Haghighi <omid.ehtemamhaghighi@menlosecurity.com>
    Cc: Shuah Khan <shuah@kernel.org>
    Cc: Ido Schimmel <idosch@idosch.org>
    Cc: Kuniyuki Iwashima <kuniyu@amazon.com>
    Cc: Simon Horman <horms@kernel.org>
    Reviewed-by: David Ahern <dsahern@kernel.org>
    Link: https://patch.msgid.link/20241106010236.1239299-1-omid.ehtemamhaghighi@menlosecurity.com
    Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Signed-off-by: Hangbin Liu <haliu@redhat.com>

Author: Hangbin Liu

net/ipv6/ip6_fib.c

@@ -1183,8 +1183,8 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt,
 		while (sibling) {
 			if (sibling->fib6_metric == rt->fib6_metric &&
 			    rt6_qualify_for_ecmp(sibling)) {
-				list_add_tail(&rt->fib6_siblings,
-					      &sibling->fib6_siblings);
+				list_add_tail_rcu(&rt->fib6_siblings,
+						  &sibling->fib6_siblings);
 				break;
 			}
 			sibling = rcu_dereference_protected(sibling->fib6_next,
@@ -1245,7 +1245,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt,
 							 fib6_siblings)
 					sibling->fib6_nsiblings--;
 				rt->fib6_nsiblings = 0;
-				list_del_init(&rt->fib6_siblings);
+				list_del_rcu(&rt->fib6_siblings);
 				rt6_multipath_rebalance(next_sibling);
 				return err;
 			}
@@ -1959,7 +1959,7 @@ static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn,
 					 &rt->fib6_siblings, fib6_siblings)
 			sibling->fib6_nsiblings--;
 		rt->fib6_nsiblings = 0;
-		list_del_init(&rt->fib6_siblings);
+		list_del_rcu(&rt->fib6_siblings);
 		rt6_multipath_rebalance(next_sibling);
 	}
 

net/ipv6/route.c

@@ -419,8 +419,8 @@ void fib6_select_path(const struct net *net, struct fib6_result *res,
 		      struct flowi6 *fl6, int oif, bool have_oif_match,
 		      const struct sk_buff *skb, int strict)
 {
-	struct fib6_info *sibling, *next_sibling;
 	struct fib6_info *match = res->f6i;
+	struct fib6_info *sibling;
 
 	if (!match->nh && (!match->fib6_nsiblings || have_oif_match))
 		goto out;
@@ -446,8 +446,8 @@ void fib6_select_path(const struct net *net, struct fib6_result *res,
 	if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound))
 		goto out;
 
-	list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings,
-				 fib6_siblings) {
+	list_for_each_entry_rcu(sibling, &match->fib6_siblings,
+				fib6_siblings) {
 		const struct fib6_nh *nh = sibling->fib6_nh;
 		int nh_upper_bound;
 
@@ -5216,14 +5216,18 @@ static void ip6_route_mpath_notify(struct fib6_info *rt,
 	 * nexthop. Since sibling routes are always added at the end of
 	 * the list, find the first sibling of the last route appended
 	 */
+	rcu_read_lock();
+
 	if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) {
-		rt = list_first_entry(&rt_last->fib6_siblings,
-				      struct fib6_info,
-				      fib6_siblings);
+		rt = list_first_or_null_rcu(&rt_last->fib6_siblings,
+					    struct fib6_info,
+					    fib6_siblings);
 	}
 
 	if (rt)
 		inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
+
+	rcu_read_unlock();
 }
 
 static bool ip6_route_mpath_should_notify(const struct fib6_info *rt)
@@ -5568,17 +5572,21 @@ static size_t rt6_nlmsg_size(struct fib6_info *f6i)
 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,
 					 &nexthop_len);
 	} else {
-		struct fib6_info *sibling, *next_sibling;
 		struct fib6_nh *nh = f6i->fib6_nh;
+		struct fib6_info *sibling;
 
 		nexthop_len = 0;
 		if (f6i->fib6_nsiblings) {
 			rt6_nh_nlmsg_size(nh, &nexthop_len);
 
-			list_for_each_entry_safe(sibling, next_sibling,
-						 &f6i->fib6_siblings, fib6_siblings) {
+			rcu_read_lock();
+
+			list_for_each_entry_rcu(sibling, &f6i->fib6_siblings,
+						fib6_siblings) {
 				rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len);
 			}
+
+			rcu_read_unlock();
 		}
 		nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);
 	}
@@ -5742,7 +5750,7 @@ static int rt6_fill_node(struct net *net, struct sk_buff *skb,
 		    lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0)
 			goto nla_put_failure;
 	} else if (rt->fib6_nsiblings) {
-		struct fib6_info *sibling, *next_sibling;
+		struct fib6_info *sibling;
 		struct nlattr *mp;
 
 		mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
@@ -5754,14 +5762,21 @@ static int rt6_fill_node(struct net *net, struct sk_buff *skb,
 				    0) < 0)
 			goto nla_put_failure;
 
-		list_for_each_entry_safe(sibling, next_sibling,
-					 &rt->fib6_siblings, fib6_siblings) {
+		rcu_read_lock();
+
+		list_for_each_entry_rcu(sibling, &rt->fib6_siblings,
+					fib6_siblings) {
 			if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common,
 					    sibling->fib6_nh->fib_nh_weight,
-					    AF_INET6, 0) < 0)
+					    AF_INET6, 0) < 0) {
+				rcu_read_unlock();
+
 				goto nla_put_failure;
+			}
 		}
 
+		rcu_read_unlock();
+
 		nla_nest_end(skb, mp);
 	} else if (rt->nh) {
 		if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))
@@ -6198,7 +6213,7 @@ void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
 	err = -ENOBUFS;
 	seq = info->nlh ? info->nlh->nlmsg_seq : 0;
 
-	skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
+	skb = nlmsg_new(rt6_nlmsg_size(rt), GFP_ATOMIC);
 	if (!skb)
 		goto errout;
 
@@ -6211,7 +6226,7 @@ void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
 		goto errout;
 	}
 	rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
-		    info->nlh, gfp_any());
+		    info->nlh, GFP_ATOMIC);
 	return;
 errout:
 	if (err < 0)

tools/testing/selftests/net/Makefile

@@ -64,6 +64,7 @@ TEST_PROGS += test_vxlan_mdb.sh
 TEST_PROGS += test_bridge_neigh_suppress.sh
 TEST_PROGS += test_vxlan_nolocalbypass.sh
 TEST_PROGS += test_bridge_backup_port.sh
+TEST_PROGS += ipv6_route_update_soft_lockup.sh
 
 TEST_FILES := settings
 TEST_FILES += in_netns.sh lib.sh net_helper.sh setup_loopback.sh setup_veth.sh