3434
3535#define xfrm_state_deref_prot (table , net ) \
3636 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37+ #define xfrm_state_deref_check (table , net ) \
38+ rcu_dereference_check((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
3739
3840static void xfrm_state_gc_task (struct work_struct * work );
3941
@@ -62,6 +64,8 @@ static inline unsigned int xfrm_dst_hash(struct net *net,
6264 u32 reqid ,
6365 unsigned short family )
6466{
67+ lockdep_assert_held (& net -> xfrm .xfrm_state_lock );
68+
6569 return __xfrm_dst_hash (daddr , saddr , reqid , family , net -> xfrm .state_hmask );
6670}
6771
@@ -70,18 +74,24 @@ static inline unsigned int xfrm_src_hash(struct net *net,
7074 const xfrm_address_t * saddr ,
7175 unsigned short family )
7276{
77+ lockdep_assert_held (& net -> xfrm .xfrm_state_lock );
78+
7379 return __xfrm_src_hash (daddr , saddr , family , net -> xfrm .state_hmask );
7480}
7581
7682static inline unsigned int
7783xfrm_spi_hash (struct net * net , const xfrm_address_t * daddr ,
7884 __be32 spi , u8 proto , unsigned short family )
7985{
86+ lockdep_assert_held (& net -> xfrm .xfrm_state_lock );
87+
8088 return __xfrm_spi_hash (daddr , spi , proto , family , net -> xfrm .state_hmask );
8189}
8290
8391static unsigned int xfrm_seq_hash (struct net * net , u32 seq )
8492{
93+ lockdep_assert_held (& net -> xfrm .xfrm_state_lock );
94+
8595 return __xfrm_seq_hash (seq , net -> xfrm .state_hmask );
8696}
8797
@@ -1032,16 +1042,38 @@ xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
10321042 x -> props .family = tmpl -> encap_family ;
10331043}
10341044
1035- static struct xfrm_state * __xfrm_state_lookup_all (struct net * net , u32 mark ,
1045+ struct xfrm_hash_state_ptrs {
1046+ const struct hlist_head * bydst ;
1047+ const struct hlist_head * bysrc ;
1048+ const struct hlist_head * byspi ;
1049+ unsigned int hmask ;
1050+ };
1051+
1052+ static void xfrm_hash_ptrs_get (const struct net * net , struct xfrm_hash_state_ptrs * ptrs )
1053+ {
1054+ unsigned int sequence ;
1055+
1056+ do {
1057+ sequence = read_seqcount_begin (& net -> xfrm .xfrm_state_hash_generation );
1058+
1059+ ptrs -> bydst = xfrm_state_deref_check (net -> xfrm .state_bydst , net );
1060+ ptrs -> bysrc = xfrm_state_deref_check (net -> xfrm .state_bysrc , net );
1061+ ptrs -> byspi = xfrm_state_deref_check (net -> xfrm .state_byspi , net );
1062+ ptrs -> hmask = net -> xfrm .state_hmask ;
1063+ } while (read_seqcount_retry (& net -> xfrm .xfrm_state_hash_generation , sequence ));
1064+ }
1065+
1066+ static struct xfrm_state * __xfrm_state_lookup_all (const struct xfrm_hash_state_ptrs * state_ptrs ,
1067+ u32 mark ,
10361068 const xfrm_address_t * daddr ,
10371069 __be32 spi , u8 proto ,
10381070 unsigned short family ,
10391071 struct xfrm_dev_offload * xdo )
10401072{
1041- unsigned int h = xfrm_spi_hash ( net , daddr , spi , proto , family );
1073+ unsigned int h = __xfrm_spi_hash ( daddr , spi , proto , family , state_ptrs -> hmask );
10421074 struct xfrm_state * x ;
10431075
1044- hlist_for_each_entry_rcu (x , net -> xfrm . state_byspi + h , byspi ) {
1076+ hlist_for_each_entry_rcu (x , state_ptrs -> byspi + h , byspi ) {
10451077#ifdef CONFIG_XFRM_OFFLOAD
10461078 if (xdo -> type == XFRM_DEV_OFFLOAD_PACKET ) {
10471079 if (x -> xso .type != XFRM_DEV_OFFLOAD_PACKET )
@@ -1075,15 +1107,16 @@ static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark,
10751107 return NULL ;
10761108}
10771109
1078- static struct xfrm_state * __xfrm_state_lookup (struct net * net , u32 mark ,
1110+ static struct xfrm_state * __xfrm_state_lookup (const struct xfrm_hash_state_ptrs * state_ptrs ,
1111+ u32 mark ,
10791112 const xfrm_address_t * daddr ,
10801113 __be32 spi , u8 proto ,
10811114 unsigned short family )
10821115{
1083- unsigned int h = xfrm_spi_hash ( net , daddr , spi , proto , family );
1116+ unsigned int h = __xfrm_spi_hash ( daddr , spi , proto , family , state_ptrs -> hmask );
10841117 struct xfrm_state * x ;
10851118
1086- hlist_for_each_entry_rcu (x , net -> xfrm . state_byspi + h , byspi ) {
1119+ hlist_for_each_entry_rcu (x , state_ptrs -> byspi + h , byspi ) {
10871120 if (x -> props .family != family ||
10881121 x -> id .spi != spi ||
10891122 x -> id .proto != proto ||
@@ -1100,15 +1133,16 @@ static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,
11001133 return NULL ;
11011134}
11021135
1103- static struct xfrm_state * __xfrm_state_lookup_byaddr (struct net * net , u32 mark ,
1136+ static struct xfrm_state * __xfrm_state_lookup_byaddr (const struct xfrm_hash_state_ptrs * state_ptrs ,
1137+ u32 mark ,
11041138 const xfrm_address_t * daddr ,
11051139 const xfrm_address_t * saddr ,
11061140 u8 proto , unsigned short family )
11071141{
1108- unsigned int h = xfrm_src_hash ( net , daddr , saddr , family );
1142+ unsigned int h = __xfrm_src_hash ( daddr , saddr , family , state_ptrs -> hmask );
11091143 struct xfrm_state * x ;
11101144
1111- hlist_for_each_entry_rcu (x , net -> xfrm . state_bysrc + h , bysrc ) {
1145+ hlist_for_each_entry_rcu (x , state_ptrs -> bysrc + h , bysrc ) {
11121146 if (x -> props .family != family ||
11131147 x -> id .proto != proto ||
11141148 !xfrm_addr_equal (& x -> id .daddr , daddr , family ) ||
@@ -1128,14 +1162,17 @@ static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark,
11281162static inline struct xfrm_state *
11291163__xfrm_state_locate (struct xfrm_state * x , int use_spi , int family )
11301164{
1165+ struct xfrm_hash_state_ptrs state_ptrs ;
11311166 struct net * net = xs_net (x );
11321167 u32 mark = x -> mark .v & x -> mark .m ;
11331168
1169+ xfrm_hash_ptrs_get (net , & state_ptrs );
1170+
11341171 if (use_spi )
1135- return __xfrm_state_lookup (net , mark , & x -> id .daddr ,
1172+ return __xfrm_state_lookup (& state_ptrs , mark , & x -> id .daddr ,
11361173 x -> id .spi , x -> id .proto , family );
11371174 else
1138- return __xfrm_state_lookup_byaddr (net , mark ,
1175+ return __xfrm_state_lookup_byaddr (& state_ptrs , mark ,
11391176 & x -> id .daddr ,
11401177 & x -> props .saddr ,
11411178 x -> id .proto , family );
@@ -1198,6 +1235,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
11981235 unsigned short family , u32 if_id )
11991236{
12001237 static xfrm_address_t saddr_wildcard = { };
1238+ struct xfrm_hash_state_ptrs state_ptrs ;
12011239 struct net * net = xp_net (pol );
12021240 unsigned int h , h_wildcard ;
12031241 struct xfrm_state * x , * x0 , * to_put ;
@@ -1214,8 +1252,10 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
12141252 sequence = read_seqcount_begin (& net -> xfrm .xfrm_state_hash_generation );
12151253
12161254 rcu_read_lock ();
1217- h = xfrm_dst_hash (net , daddr , saddr , tmpl -> reqid , encap_family );
1218- hlist_for_each_entry_rcu (x , net -> xfrm .state_bydst + h , bydst ) {
1255+ xfrm_hash_ptrs_get (net , & state_ptrs );
1256+
1257+ h = __xfrm_dst_hash (daddr , saddr , tmpl -> reqid , encap_family , state_ptrs .hmask );
1258+ hlist_for_each_entry_rcu (x , state_ptrs .bydst + h , bydst ) {
12191259#ifdef CONFIG_XFRM_OFFLOAD
12201260 if (pol -> xdo .type == XFRM_DEV_OFFLOAD_PACKET ) {
12211261 if (x -> xso .type != XFRM_DEV_OFFLOAD_PACKET )
@@ -1248,8 +1288,9 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
12481288 if (best || acquire_in_progress )
12491289 goto found ;
12501290
1251- h_wildcard = xfrm_dst_hash (net , daddr , & saddr_wildcard , tmpl -> reqid , encap_family );
1252- hlist_for_each_entry_rcu (x , net -> xfrm .state_bydst + h_wildcard , bydst ) {
1291+ h_wildcard = __xfrm_dst_hash (daddr , & saddr_wildcard , tmpl -> reqid ,
1292+ encap_family , state_ptrs .hmask );
1293+ hlist_for_each_entry_rcu (x , state_ptrs .bydst + h_wildcard , bydst ) {
12531294#ifdef CONFIG_XFRM_OFFLOAD
12541295 if (pol -> xdo .type == XFRM_DEV_OFFLOAD_PACKET ) {
12551296 if (x -> xso .type != XFRM_DEV_OFFLOAD_PACKET )
@@ -1284,7 +1325,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
12841325 x = best ;
12851326 if (!x && !error && !acquire_in_progress ) {
12861327 if (tmpl -> id .spi &&
1287- (x0 = __xfrm_state_lookup_all (net , mark , daddr ,
1328+ (x0 = __xfrm_state_lookup_all (& state_ptrs , mark , daddr ,
12881329 tmpl -> id .spi , tmpl -> id .proto ,
12891330 encap_family ,
12901331 & pol -> xdo )) != NULL ) {
@@ -2030,10 +2071,13 @@ struct xfrm_state *
20302071xfrm_state_lookup (struct net * net , u32 mark , const xfrm_address_t * daddr , __be32 spi ,
20312072 u8 proto , unsigned short family )
20322073{
2074+ struct xfrm_hash_state_ptrs state_ptrs ;
20332075 struct xfrm_state * x ;
20342076
20352077 rcu_read_lock ();
2036- x = __xfrm_state_lookup (net , mark , daddr , spi , proto , family );
2078+ xfrm_hash_ptrs_get (net , & state_ptrs );
2079+
2080+ x = __xfrm_state_lookup (& state_ptrs , mark , daddr , spi , proto , family );
20372081 rcu_read_unlock ();
20382082 return x ;
20392083}
@@ -2044,10 +2088,14 @@ xfrm_state_lookup_byaddr(struct net *net, u32 mark,
20442088 const xfrm_address_t * daddr , const xfrm_address_t * saddr ,
20452089 u8 proto , unsigned short family )
20462090{
2091+ struct xfrm_hash_state_ptrs state_ptrs ;
20472092 struct xfrm_state * x ;
20482093
20492094 spin_lock_bh (& net -> xfrm .xfrm_state_lock );
2050- x = __xfrm_state_lookup_byaddr (net , mark , daddr , saddr , proto , family );
2095+
2096+ xfrm_hash_ptrs_get (net , & state_ptrs );
2097+
2098+ x = __xfrm_state_lookup_byaddr (& state_ptrs , mark , daddr , saddr , proto , family );
20512099 spin_unlock_bh (& net -> xfrm .xfrm_state_lock );
20522100 return x ;
20532101}
0 commit comments