page_pool: Move pp_magic check into helper functions

jira LE-3187
Rebuild_History Non-Buildable kernel-5.14.0-570.19.1.el9_6
commit-author Toke Høiland-Jørgensen <toke@redhat.com>
commit cd3c931
Empty-Commit: Cherry-Pick Conflicts during history rebuild.
Will be included in final tarball splat. Ref for failed cherry-pick at:
ciq/ciq_backports/kernel-5.14.0-570.19.1.el9_6/cd3c9316.failed

Since we are about to stash some more information into the pp_magic
field, let's move the magic signature checks into a pair of helper
functions so it can be changed in one place.

	Reviewed-by: Mina Almasry <almasrymina@google.com>
	Tested-by: Yonglong Liu <liuyonglong@huawei.com>
	Acked-by: Jesper Dangaard Brouer <hawk@kernel.org>
	Reviewed-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
	Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://patch.msgid.link/20250409-page-pool-track-dma-v9-1-6a9ef2e0cba8@redhat.com
	Signed-off-by: Jakub Kicinski <kuba@kernel.org>
(cherry picked from commit cd3c931)
	Signed-off-by: Jonathan Maple <jmaple@ciq.com>

# Conflicts:
#	include/linux/mm.h
#	mm/page_alloc.c
#	net/core/netmem_priv.h
#	net/core/skbuff.c

Author: PlaidCat

ciq/ciq_backports/kernel-5.14.0-570.19.1.el9_6/cd3c9316.failed

@@ -0,0 +1,356 @@
+page_pool: Move pp_magic check into helper functions
+
+jira LE-3187
+Rebuild_History Non-Buildable kernel-5.14.0-570.19.1.el9_6
+commit-author Toke Høiland-Jørgensen <toke@redhat.com>
+commit cd3c93167da0e760b5819246eae7a4ea30fd014b
+Empty-Commit: Cherry-Pick Conflicts during history rebuild.
+Will be included in final tarball splat. Ref for failed cherry-pick at:
+ciq/ciq_backports/kernel-5.14.0-570.19.1.el9_6/cd3c9316.failed
+
+Since we are about to stash some more information into the pp_magic
+field, let's move the magic signature checks into a pair of helper
+functions so it can be changed in one place.
+
+	Reviewed-by: Mina Almasry <almasrymina@google.com>
+	Tested-by: Yonglong Liu <liuyonglong@huawei.com>
+	Acked-by: Jesper Dangaard Brouer <hawk@kernel.org>
+	Reviewed-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
+	Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
+Link: https://patch.msgid.link/20250409-page-pool-track-dma-v9-1-6a9ef2e0cba8@redhat.com
+	Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+(cherry picked from commit cd3c93167da0e760b5819246eae7a4ea30fd014b)
+	Signed-off-by: Jonathan Maple <jmaple@ciq.com>
+
+# Conflicts:
+#	include/linux/mm.h
+#	mm/page_alloc.c
+#	net/core/netmem_priv.h
+#	net/core/skbuff.c
+diff --cc include/linux/mm.h
+index 5d9b789ea0a7,56c47f4a38ca..000000000000
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@@ -4138,9 -4198,74 +4138,79 @@@ static inline void accept_memory(phys_a
+  
+  static inline bool pfn_is_unaccepted_memory(unsigned long pfn)
+  {
+ -	return range_contains_unaccepted_memory(pfn << PAGE_SHIFT, PAGE_SIZE);
+ +	phys_addr_t paddr = pfn << PAGE_SHIFT;
+ +
+ +	return range_contains_unaccepted_memory(paddr, paddr + PAGE_SIZE);
+  }
+  
+++<<<<<<< HEAD
+++=======
++ void vma_pgtable_walk_begin(struct vm_area_struct *vma);
++ void vma_pgtable_walk_end(struct vm_area_struct *vma);
++ 
++ int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size);
++ int reserve_mem_release_by_name(const char *name);
++ 
++ #ifdef CONFIG_64BIT
++ int do_mseal(unsigned long start, size_t len_in, unsigned long flags);
++ #else
++ static inline int do_mseal(unsigned long start, size_t len_in, unsigned long flags)
++ {
++ 	/* noop on 32 bit */
++ 	return 0;
++ }
++ #endif
++ 
++ /*
++  * user_alloc_needs_zeroing checks if a user folio from page allocator needs to
++  * be zeroed or not.
++  */
++ static inline bool user_alloc_needs_zeroing(void)
++ {
++ 	/*
++ 	 * for user folios, arch with cache aliasing requires cache flush and
++ 	 * arc changes folio->flags to make icache coherent with dcache, so
++ 	 * always return false to make caller use
++ 	 * clear_user_page()/clear_user_highpage().
++ 	 */
++ 	return cpu_dcache_is_aliasing() || cpu_icache_is_aliasing() ||
++ 	       !static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
++ 				   &init_on_alloc);
++ }
++ 
++ int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status);
++ int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status);
++ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
++ 
++ 
++ /*
++  * mseal of userspace process's system mappings.
++  */
++ #ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS
++ #define VM_SEALED_SYSMAP	VM_SEALED
++ #else
++ #define VM_SEALED_SYSMAP	VM_NONE
++ #endif
++ 
++ /* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is
++  * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for
++  * the head page of compound page and bit 1 for pfmemalloc page.
++  * page_is_pfmemalloc() is checked in __page_pool_put_page() to avoid recycling
++  * the pfmemalloc page.
++  */
++ #define PP_MAGIC_MASK ~0x3UL
++ 
++ #ifdef CONFIG_PAGE_POOL
++ static inline bool page_pool_page_is_pp(struct page *page)
++ {
++ 	return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE;
++ }
++ #else
++ static inline bool page_pool_page_is_pp(struct page *page)
++ {
++ 	return false;
++ }
++ #endif
++ 
+++>>>>>>> cd3c93167da0 (page_pool: Move pp_magic check into helper functions)
+  #endif /* _LINUX_MM_H */
+diff --cc mm/page_alloc.c
+index aa2cbab0e18e,a18340b32218..000000000000
+--- a/mm/page_alloc.c
++++ b/mm/page_alloc.c
+@@@ -919,6 -897,7 +919,10 @@@ static inline bool page_expected_state(
+  #ifdef CONFIG_MEMCG
+  			page->memcg_data |
+  #endif
+++<<<<<<< HEAD
+++=======
++ 			page_pool_page_is_pp(page) |
+++>>>>>>> cd3c93167da0 (page_pool: Move pp_magic check into helper functions)
+  			(page->flags & check_flags)))
+  		return false;
+  
+@@@ -945,6 -924,8 +949,11 @@@ static const char *page_bad_reason(stru
+  	if (unlikely(page->memcg_data))
+  		bad_reason = "page still charged to cgroup";
+  #endif
+++<<<<<<< HEAD
+++=======
++ 	if (unlikely(page_pool_page_is_pp(page)))
++ 		bad_reason = "page_pool leak";
+++>>>>>>> cd3c93167da0 (page_pool: Move pp_magic check into helper functions)
+  	return bad_reason;
+  }
+  
+diff --cc net/core/skbuff.c
+index ad137d532882,74a2d886a35b..000000000000
+--- a/net/core/skbuff.c
++++ b/net/core/skbuff.c
+@@@ -855,22 -893,107 +855,32 @@@ static void skb_clone_fraglist(struct s
+  		skb_get(list);
+  }
+  
+++<<<<<<< HEAD
+++=======
++ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
++ 		    unsigned int headroom)
++ {
+++>>>>>>> cd3c93167da0 (page_pool: Move pp_magic check into helper functions)
+  #if IS_ENABLED(CONFIG_PAGE_POOL)
+ -	u32 size, truesize, len, max_head_size, off;
+ -	struct sk_buff *skb = *pskb, *nskb;
+ -	int err, i, head_off;
+ -	void *data;
+ -
+ -	/* XDP does not support fraglist so we need to linearize
+ -	 * the skb.
+ -	 */
+ -	if (skb_has_frag_list(skb))
+ -		return -EOPNOTSUPP;
+ -
+ -	max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom);
+ -	if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE)
+ -		return -ENOMEM;
+ -
+ -	size = min_t(u32, skb->len, max_head_size);
+ -	truesize = SKB_HEAD_ALIGN(size) + headroom;
+ -	data = page_pool_dev_alloc_va(pool, &truesize);
+ -	if (!data)
+ -		return -ENOMEM;
+ -
+ -	nskb = napi_build_skb(data, truesize);
+ -	if (!nskb) {
+ -		page_pool_free_va(pool, data, true);
+ -		return -ENOMEM;
+ -	}
+ -
+ -	skb_reserve(nskb, headroom);
+ -	skb_copy_header(nskb, skb);
+ -	skb_mark_for_recycle(nskb);
+ -
+ -	err = skb_copy_bits(skb, 0, nskb->data, size);
+ -	if (err) {
+ -		consume_skb(nskb);
+ -		return err;
+ -	}
+ -	skb_put(nskb, size);
+ -
+ -	head_off = skb_headroom(nskb) - skb_headroom(skb);
+ -	skb_headers_offset_update(nskb, head_off);
+ -
+ -	off = size;
+ -	len = skb->len - off;
+ -	for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
+ -		struct page *page;
+ -		u32 page_off;
+ -
+ -		size = min_t(u32, len, PAGE_SIZE);
+ -		truesize = size;
+ -
+ -		page = page_pool_dev_alloc(pool, &page_off, &truesize);
+ -		if (!page) {
+ -			consume_skb(nskb);
+ -			return -ENOMEM;
+ -		}
+ -
+ -		skb_add_rx_frag(nskb, i, page, page_off, size, truesize);
+ -		err = skb_copy_bits(skb, off, page_address(page) + page_off,
+ -				    size);
+ -		if (err) {
+ -			consume_skb(nskb);
+ -			return err;
+ -		}
+ -
+ -		len -= size;
+ -		off += size;
+ -	}
+ -
+ -	consume_skb(skb);
+ -	*pskb = nskb;
+ -
+ -	return 0;
+ -#else
+ -	return -EOPNOTSUPP;
+ -#endif
+ -}
+ -EXPORT_SYMBOL(skb_pp_cow_data);
+ -
+ -int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
+ -			 const struct bpf_prog *prog)
+ -{
+ -	if (!prog->aux->xdp_has_frags)
+ -		return -EINVAL;
+ -
+ -	return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM);
+ -}
+ -EXPORT_SYMBOL(skb_cow_data_for_xdp);
+ -
+ -#if IS_ENABLED(CONFIG_PAGE_POOL)
+ -bool napi_pp_put_page(netmem_ref netmem)
+ +bool napi_pp_put_page(struct page *page)
+  {
+ -	netmem = netmem_compound_head(netmem);
+ +	page = compound_head(page);
+  
+++<<<<<<< HEAD
+ +	/* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation
+ +	 * in order to preserve any existing bits, such as bit 0 for the
+ +	 * head page of compound page and bit 1 for pfmemalloc page, so
+ +	 * mask those bits for freeing side when doing below checking,
+ +	 * and page_is_pfmemalloc() is checked in __page_pool_put_page()
+ +	 * to avoid recycling the pfmemalloc page.
+ +	 */
+ +	if (unlikely((page->pp_magic & ~0x3UL) != PP_SIGNATURE))
+++=======
++ 	if (unlikely(!netmem_is_pp(netmem)))
+++>>>>>>> cd3c93167da0 (page_pool: Move pp_magic check into helper functions)
+  		return false;
+  
+ -	page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
+ +	page_pool_put_full_page(page->pp, page, false);
+  
+  	return true;
+  }
+@@@ -881,7 -1004,46 +891,50 @@@ static bool skb_pp_recycle(struct sk_bu
+  {
+  	if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
+  		return false;
+++<<<<<<< HEAD
+ +	return napi_pp_put_page(virt_to_page(data));
+++=======
++ 	return napi_pp_put_page(page_to_netmem(virt_to_page(data)));
++ }
++ 
++ /**
++  * skb_pp_frag_ref() - Increase fragment references of a page pool aware skb
++  * @skb:	page pool aware skb
++  *
++  * Increase the fragment reference count (pp_ref_count) of a skb. This is
++  * intended to gain fragment references only for page pool aware skbs,
++  * i.e. when skb->pp_recycle is true, and not for fragments in a
++  * non-pp-recycling skb. It has a fallback to increase references on normal
++  * pages, as page pool aware skbs may also have normal page fragments.
++  */
++ static int skb_pp_frag_ref(struct sk_buff *skb)
++ {
++ 	struct skb_shared_info *shinfo;
++ 	netmem_ref head_netmem;
++ 	int i;
++ 
++ 	if (!skb->pp_recycle)
++ 		return -EINVAL;
++ 
++ 	shinfo = skb_shinfo(skb);
++ 
++ 	for (i = 0; i < shinfo->nr_frags; i++) {
++ 		head_netmem = netmem_compound_head(shinfo->frags[i].netmem);
++ 		if (likely(netmem_is_pp(head_netmem)))
++ 			page_pool_ref_netmem(head_netmem);
++ 		else
++ 			page_ref_inc(netmem_to_page(head_netmem));
++ 	}
++ 	return 0;
++ }
++ 
++ static void skb_kfree_head(void *head, unsigned int end_offset)
++ {
++ 	if (end_offset == SKB_SMALL_HEAD_HEADROOM)
++ 		kmem_cache_free(net_hotdata.skb_small_head_cache, head);
++ 	else
++ 		kfree(head);
+++>>>>>>> cd3c93167da0 (page_pool: Move pp_magic check into helper functions)
+  }
+  
+  static void skb_free_head(struct sk_buff *skb)
+* Unmerged path net/core/netmem_priv.h
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+index bb7a60a3251b..9ac1f586b17b 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+@@ -655,8 +655,8 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
+ 				xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+ 				page = xdpi.page.page;
+ 
+-				/* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE)
+-				 * as we know this is a page_pool page.
++				/* No need to check page_pool_page_is_pp() as we
++				 * know this is a page_pool page.
+ 				 */
+ 				page_pool_recycle_direct(page->pp, page);
+ 			} while (++n < num);
+* Unmerged path include/linux/mm.h
+* Unmerged path mm/page_alloc.c
+* Unmerged path net/core/netmem_priv.h
+* Unmerged path net/core/skbuff.c
+diff --git a/net/core/xdp.c b/net/core/xdp.c
+index 04d61dc16544..4836433b5033 100644
+--- a/net/core/xdp.c
++++ b/net/core/xdp.c
+@@ -380,8 +380,8 @@ void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
+ 		page = virt_to_head_page(data);
+ 		if (napi_direct && xdp_return_frame_no_direct())
+ 			napi_direct = false;
+-		/* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE)
+-		 * as mem->type knows this a page_pool page
++		/* No need to check netmem_is_pp() as mem->type knows this a
++		 * page_pool page
+ 		 */
+ 		page_pool_put_full_page(page->pp, page, napi_direct);
+ 		break;