@@ -894,10 +894,6 @@ ice_add_xdp_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
894894 __skb_fill_page_desc_noacc (sinfo , sinfo -> nr_frags ++ , rx_buf -> page ,
895895 rx_buf -> page_offset , size );
896896 sinfo -> xdp_frags_size += size ;
897- /* remember frag count before XDP prog execution; bpf_xdp_adjust_tail()
898- * can pop off frags but driver has to handle it on its own
899- */
900- rx_ring -> nr_frags = sinfo -> nr_frags ;
901897
902898 if (page_is_pfmemalloc (rx_buf -> page ))
903899 xdp_buff_set_frag_pfmemalloc (xdp );
@@ -968,20 +964,20 @@ ice_get_rx_buf(struct ice_rx_ring *rx_ring, const unsigned int size,
968964/**
969965 * ice_get_pgcnts - grab page_count() for gathered fragments
970966 * @rx_ring: Rx descriptor ring to store the page counts on
967+ * @ntc: the next to clean element (not included in this frame!)
971968 *
972969 * This function is intended to be called right before running XDP
973970 * program so that the page recycling mechanism will be able to take
974971 * a correct decision regarding underlying pages; this is done in such
975972 * way as XDP program can change the refcount of page
976973 */
977- static void ice_get_pgcnts (struct ice_rx_ring * rx_ring )
974+ static void ice_get_pgcnts (struct ice_rx_ring * rx_ring , unsigned int ntc )
978975{
979- u32 nr_frags = rx_ring -> nr_frags + 1 ;
980976 u32 idx = rx_ring -> first_desc ;
981977 struct ice_rx_buf * rx_buf ;
982978 u32 cnt = rx_ring -> count ;
983979
984- for ( int i = 0 ; i < nr_frags ; i ++ ) {
980+ while ( idx != ntc ) {
985981 rx_buf = & rx_ring -> rx_buf [idx ];
986982 rx_buf -> pgcnt = page_count (rx_buf -> page );
987983
@@ -1154,62 +1150,51 @@ ice_put_rx_buf(struct ice_rx_ring *rx_ring, struct ice_rx_buf *rx_buf)
11541150}
11551151
11561152/**
1157- * ice_put_rx_mbuf - ice_put_rx_buf() caller, for all frame frags
1153+ * ice_put_rx_mbuf - ice_put_rx_buf() caller, for all buffers in frame
11581154 * @rx_ring: Rx ring with all the auxiliary data
11591155 * @xdp: XDP buffer carrying linear + frags part
1160- * @xdp_xmit: XDP_TX/XDP_REDIRECT verdict storage
1161- * @ntc: a current next_to_clean value to be stored at rx_ring
1156+ * @ntc: the next to clean element (not included in this frame!)
11621157 * @verdict: return code from XDP program execution
11631158 *
1164- * Walk through gathered fragments and satisfy internal page
1165- * recycle mechanism; we take here an action related to verdict
1166- * returned by XDP program;
1159+ * Called after XDP program is completed, or on error with verdict set to
1160+ * ICE_XDP_CONSUMED.
1161+ *
1162+ * Walk through buffers from first_desc to the end of the frame, releasing
1163+ * buffers and satisfying internal page recycle mechanism. The action depends
1164+ * on verdict from XDP program.
11671165 */
11681166static void ice_put_rx_mbuf (struct ice_rx_ring * rx_ring , struct xdp_buff * xdp ,
1169- u32 * xdp_xmit , u32 ntc , u32 verdict )
1167+ u32 ntc , u32 verdict )
11701168{
1171- u32 nr_frags = rx_ring -> nr_frags + 1 ;
11721169 u32 idx = rx_ring -> first_desc ;
11731170 u32 cnt = rx_ring -> count ;
1174- u32 post_xdp_frags = 1 ;
11751171 struct ice_rx_buf * buf ;
1176- int i ;
1172+ u32 xdp_frags = 0 ;
1173+ int i = 0 ;
11771174
11781175 if (unlikely (xdp_buff_has_frags (xdp )))
1179- post_xdp_frags + = xdp_get_shared_info_from_buff (xdp )-> nr_frags ;
1176+ xdp_frags = xdp_get_shared_info_from_buff (xdp )-> nr_frags ;
11801177
1181- for ( i = 0 ; i < post_xdp_frags ; i ++ ) {
1178+ while ( idx != ntc ) {
11821179 buf = & rx_ring -> rx_buf [idx ];
1180+ if (++ idx == cnt )
1181+ idx = 0 ;
11831182
1184- if (verdict & (ICE_XDP_TX | ICE_XDP_REDIR )) {
1183+ /* An XDP program could release fragments from the end of the
1184+ * buffer. For these, we need to keep the pagecnt_bias as-is.
1185+ * To do this, only adjust pagecnt_bias for fragments up to
1186+ * the total remaining after the XDP program has run.
1187+ */
1188+ if (verdict != ICE_XDP_CONSUMED )
11851189 ice_rx_buf_adjust_pg_offset (buf , xdp -> frame_sz );
1186- * xdp_xmit |= verdict ;
1187- } else if (verdict & ICE_XDP_CONSUMED ) {
1190+ else if (i ++ <= xdp_frags )
11881191 buf -> pagecnt_bias ++ ;
1189- } else if (verdict == ICE_XDP_PASS ) {
1190- ice_rx_buf_adjust_pg_offset (buf , xdp -> frame_sz );
1191- }
11921192
11931193 ice_put_rx_buf (rx_ring , buf );
1194-
1195- if (++ idx == cnt )
1196- idx = 0 ;
1197- }
1198- /* handle buffers that represented frags released by XDP prog;
1199- * for these we keep pagecnt_bias as-is; refcount from struct page
1200- * has been decremented within XDP prog and we do not have to increase
1201- * the biased refcnt
1202- */
1203- for (; i < nr_frags ; i ++ ) {
1204- buf = & rx_ring -> rx_buf [idx ];
1205- ice_put_rx_buf (rx_ring , buf );
1206- if (++ idx == cnt )
1207- idx = 0 ;
12081194 }
12091195
12101196 xdp -> data = NULL ;
12111197 rx_ring -> first_desc = ntc ;
1212- rx_ring -> nr_frags = 0 ;
12131198}
12141199
12151200/**
@@ -1317,6 +1302,10 @@ static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
13171302 /* retrieve a buffer from the ring */
13181303 rx_buf = ice_get_rx_buf (rx_ring , size , ntc );
13191304
1305+ /* Increment ntc before calls to ice_put_rx_mbuf() */
1306+ if (++ ntc == cnt )
1307+ ntc = 0 ;
1308+
13201309 if (!xdp -> data ) {
13211310 void * hard_start ;
13221311
@@ -1325,24 +1314,23 @@ static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
13251314 xdp_prepare_buff (xdp , hard_start , offset , size , !!offset );
13261315 xdp_buff_clear_frags_flag (xdp );
13271316 } else if (ice_add_xdp_frag (rx_ring , xdp , rx_buf , size )) {
1328- ice_put_rx_mbuf (rx_ring , xdp , NULL , ntc , ICE_XDP_CONSUMED );
1317+ ice_put_rx_mbuf (rx_ring , xdp , ntc , ICE_XDP_CONSUMED );
13291318 break ;
13301319 }
1331- if (++ ntc == cnt )
1332- ntc = 0 ;
13331320
13341321 /* skip if it is NOP desc */
13351322 if (ice_is_non_eop (rx_ring , rx_desc ))
13361323 continue ;
13371324
1338- ice_get_pgcnts (rx_ring );
1325+ ice_get_pgcnts (rx_ring , ntc );
13391326 xdp_verdict = ice_run_xdp (rx_ring , xdp , xdp_prog , xdp_ring , rx_desc );
13401327 if (xdp_verdict == ICE_XDP_PASS )
13411328 goto construct_skb ;
13421329 total_rx_bytes += xdp_get_buff_len (xdp );
13431330 total_rx_pkts ++ ;
13441331
1345- ice_put_rx_mbuf (rx_ring , xdp , & xdp_xmit , ntc , xdp_verdict );
1332+ ice_put_rx_mbuf (rx_ring , xdp , ntc , xdp_verdict );
1333+ xdp_xmit |= xdp_verdict & (ICE_XDP_TX | ICE_XDP_REDIR );
13461334
13471335 continue ;
13481336construct_skb :
@@ -1355,7 +1343,7 @@ static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
13551343 rx_ring -> ring_stats -> rx_stats .alloc_buf_failed ++ ;
13561344 xdp_verdict = ICE_XDP_CONSUMED ;
13571345 }
1358- ice_put_rx_mbuf (rx_ring , xdp , & xdp_xmit , ntc , xdp_verdict );
1346+ ice_put_rx_mbuf (rx_ring , xdp , ntc , xdp_verdict );
13591347
13601348 if (!skb )
13611349 break ;
0 commit comments