Skip to content

Commit 95082e6

Browse files
Gbell26Gbell26
committed
bpf: Introduce load-acquire and store-release instructions
JIRA: https://issues.redhat.com/browse/RHEL-78202 commit 8804423 Author: Peilin Ye <yepeilin@google.com> Date: Tue Mar 4 01:06:13 2025 +0000 bpf: Introduce load-acquire and store-release instructions Introduce BPF instructions with load-acquire and store-release semantics, as discussed in [1]. Define 2 new flags: #define BPF_LOAD_ACQ 0x100 #define BPF_STORE_REL 0x110 A "load-acquire" is a BPF_STX | BPF_ATOMIC instruction with the 'imm' field set to BPF_LOAD_ACQ (0x100). Similarly, a "store-release" is a BPF_STX | BPF_ATOMIC instruction with the 'imm' field set to BPF_STORE_REL (0x110). Unlike existing atomic read-modify-write operations that only support BPF_W (32-bit) and BPF_DW (64-bit) size modifiers, load-acquires and store-releases also support BPF_B (8-bit) and BPF_H (16-bit). As an exception, however, 64-bit load-acquires/store-releases are not supported on 32-bit architectures (to fix a build error reported by the kernel test robot). An 8- or 16-bit load-acquire zero-extends the value before writing it to a 32-bit register, just like ARM64 instruction LDARH and friends. Similar to existing atomic read-modify-write operations, misaligned load-acquires/store-releases are not allowed (even if BPF_F_ANY_ALIGNMENT is set). As an example, consider the following 64-bit load-acquire BPF instruction (assuming little-endian): db 10 00 00 00 01 00 00 r0 = load_acquire((u64 *)(r1 + 0x0)) opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX imm (0x00000100): BPF_LOAD_ACQ Similarly, a 16-bit BPF store-release: cb 21 00 00 10 01 00 00 store_release((u16 *)(r1 + 0x0), w2) opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX imm (0x00000110): BPF_STORE_REL In arch/{arm64,s390,x86}/net/bpf_jit_comp.c, have bpf_jit_supports_insn(..., /*in_arena=*/true) return false for the new instructions, until the corresponding JIT compiler supports them in arena. [1] https://lore.kernel.org/all/20240729183246.4110549-1-yepeilin@google.com/ Acked-by: Eduard Zingerman <eddyz87@gmail.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Cc: kernel test robot <lkp@intel.com> Signed-off-by: Peilin Ye <yepeilin@google.com> Link: https://lore.kernel.org/r/a217f46f0e445fbd573a1a024be5c6bf1d5fe716.1741049567.git.yepeilin@google.com Signed-off-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: Gregory Bell <grbell@redhat.com>
1 parent dc0b8e2 commit 95082e6

File tree

10 files changed

+166
-13
lines changed

10 files changed

+166
-13
lines changed

‎arch/arm64/net/bpf_jit_comp.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2716,8 +2716,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
27162716
if (!in_arena)
27172717
return true;
27182718
switch (insn->code) {
2719+
case BPF_STX | BPF_ATOMIC | BPF_B:
2720+
case BPF_STX | BPF_ATOMIC | BPF_H:
27192721
case BPF_STX | BPF_ATOMIC | BPF_W:
27202722
case BPF_STX | BPF_ATOMIC | BPF_DW:
2723+
if (bpf_atomic_is_load_store(insn))
2724+
return false;
27212725
if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
27222726
return false;
27232727
}

‎arch/s390/net/bpf_jit_comp.c‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2919,10 +2919,16 @@ bool bpf_jit_supports_arena(void)
29192919

29202920
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
29212921
{
2922-
/*
2923-
* Currently the verifier uses this function only to check which
2924-
* atomic stores to arena are supported, and they all are.
2925-
*/
2922+
if (!in_arena)
2923+
return true;
2924+
switch (insn->code) {
2925+
case BPF_STX | BPF_ATOMIC | BPF_B:
2926+
case BPF_STX | BPF_ATOMIC | BPF_H:
2927+
case BPF_STX | BPF_ATOMIC | BPF_W:
2928+
case BPF_STX | BPF_ATOMIC | BPF_DW:
2929+
if (bpf_atomic_is_load_store(insn))
2930+
return false;
2931+
}
29262932
return true;
29272933
}
29282934

‎arch/x86/net/bpf_jit_comp.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3825,8 +3825,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
38253825
if (!in_arena)
38263826
return true;
38273827
switch (insn->code) {
3828+
case BPF_STX | BPF_ATOMIC | BPF_B:
3829+
case BPF_STX | BPF_ATOMIC | BPF_H:
38283830
case BPF_STX | BPF_ATOMIC | BPF_W:
38293831
case BPF_STX | BPF_ATOMIC | BPF_DW:
3832+
if (bpf_atomic_is_load_store(insn))
3833+
return false;
38303834
if (insn->imm == (BPF_AND | BPF_FETCH) ||
38313835
insn->imm == (BPF_OR | BPF_FETCH) ||
38323836
insn->imm == (BPF_XOR | BPF_FETCH))

‎include/linux/bpf.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -991,6 +991,21 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
991991
return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
992992
}
993993

994+
/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
995+
* atomic load or store, and false if it is a read-modify-write instruction.
996+
*/
997+
static inline bool
998+
bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
999+
{
1000+
switch (atomic_insn->imm) {
1001+
case BPF_LOAD_ACQ:
1002+
case BPF_STORE_REL:
1003+
return true;
1004+
default:
1005+
return false;
1006+
}
1007+
}
1008+
9941009
struct bpf_prog_ops {
9951010
int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
9961011
union bpf_attr __user *uattr);

‎include/linux/filter.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,8 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
364364
* BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
365365
* BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
366366
* BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
367+
* BPF_LOAD_ACQ dst_reg = smp_load_acquire(src_reg + off16)
368+
* BPF_STORE_REL smp_store_release(dst_reg + off16, src_reg)
367369
*/
368370

369371
#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \

‎include/uapi/linux/bpf.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@
5151
#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */
5252
#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */
5353

54+
#define BPF_LOAD_ACQ 0x100 /* load-acquire */
55+
#define BPF_STORE_REL 0x110 /* store-release */
56+
5457
enum bpf_cond_pseudo_jmp {
5558
BPF_MAY_GOTO = 0,
5659
};

‎kernel/bpf/core.c‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1668,14 +1668,17 @@ EXPORT_SYMBOL_GPL(__bpf_call_base);
16681668
INSN_3(JMP, JSET, K), \
16691669
INSN_2(JMP, JA), \
16701670
INSN_2(JMP32, JA), \
1671+
/* Atomic operations. */ \
1672+
INSN_3(STX, ATOMIC, B), \
1673+
INSN_3(STX, ATOMIC, H), \
1674+
INSN_3(STX, ATOMIC, W), \
1675+
INSN_3(STX, ATOMIC, DW), \
16711676
/* Store instructions. */ \
16721677
/* Register based. */ \
16731678
INSN_3(STX, MEM, B), \
16741679
INSN_3(STX, MEM, H), \
16751680
INSN_3(STX, MEM, W), \
16761681
INSN_3(STX, MEM, DW), \
1677-
INSN_3(STX, ATOMIC, W), \
1678-
INSN_3(STX, ATOMIC, DW), \
16791682
/* Immediate based. */ \
16801683
INSN_3(ST, MEM, B), \
16811684
INSN_3(ST, MEM, H), \
@@ -2157,24 +2160,33 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
21572160
if (BPF_SIZE(insn->code) == BPF_W) \
21582161
atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \
21592162
(DST + insn->off)); \
2160-
else \
2163+
else if (BPF_SIZE(insn->code) == BPF_DW) \
21612164
atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \
21622165
(DST + insn->off)); \
2166+
else \
2167+
goto default_label; \
21632168
break; \
21642169
case BOP | BPF_FETCH: \
21652170
if (BPF_SIZE(insn->code) == BPF_W) \
21662171
SRC = (u32) atomic_fetch_##KOP( \
21672172
(u32) SRC, \
21682173
(atomic_t *)(unsigned long) (DST + insn->off)); \
2169-
else \
2174+
else if (BPF_SIZE(insn->code) == BPF_DW) \
21702175
SRC = (u64) atomic64_fetch_##KOP( \
21712176
(u64) SRC, \
21722177
(atomic64_t *)(unsigned long) (DST + insn->off)); \
2178+
else \
2179+
goto default_label; \
21732180
break;
21742181

21752182
STX_ATOMIC_DW:
21762183
STX_ATOMIC_W:
2184+
STX_ATOMIC_H:
2185+
STX_ATOMIC_B:
21772186
switch (IMM) {
2187+
/* Atomic read-modify-write instructions support only W and DW
2188+
* size modifiers.
2189+
*/
21782190
ATOMIC_ALU_OP(BPF_ADD, add)
21792191
ATOMIC_ALU_OP(BPF_AND, and)
21802192
ATOMIC_ALU_OP(BPF_OR, or)
@@ -2186,20 +2198,63 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
21862198
SRC = (u32) atomic_xchg(
21872199
(atomic_t *)(unsigned long) (DST + insn->off),
21882200
(u32) SRC);
2189-
else
2201+
else if (BPF_SIZE(insn->code) == BPF_DW)
21902202
SRC = (u64) atomic64_xchg(
21912203
(atomic64_t *)(unsigned long) (DST + insn->off),
21922204
(u64) SRC);
2205+
else
2206+
goto default_label;
21932207
break;
21942208
case BPF_CMPXCHG:
21952209
if (BPF_SIZE(insn->code) == BPF_W)
21962210
BPF_R0 = (u32) atomic_cmpxchg(
21972211
(atomic_t *)(unsigned long) (DST + insn->off),
21982212
(u32) BPF_R0, (u32) SRC);
2199-
else
2213+
else if (BPF_SIZE(insn->code) == BPF_DW)
22002214
BPF_R0 = (u64) atomic64_cmpxchg(
22012215
(atomic64_t *)(unsigned long) (DST + insn->off),
22022216
(u64) BPF_R0, (u64) SRC);
2217+
else
2218+
goto default_label;
2219+
break;
2220+
/* Atomic load and store instructions support all size
2221+
* modifiers.
2222+
*/
2223+
case BPF_LOAD_ACQ:
2224+
switch (BPF_SIZE(insn->code)) {
2225+
#define LOAD_ACQUIRE(SIZEOP, SIZE) \
2226+
case BPF_##SIZEOP: \
2227+
DST = (SIZE)smp_load_acquire( \
2228+
(SIZE *)(unsigned long)(SRC + insn->off)); \
2229+
break;
2230+
LOAD_ACQUIRE(B, u8)
2231+
LOAD_ACQUIRE(H, u16)
2232+
LOAD_ACQUIRE(W, u32)
2233+
#ifdef CONFIG_64BIT
2234+
LOAD_ACQUIRE(DW, u64)
2235+
#endif
2236+
#undef LOAD_ACQUIRE
2237+
default:
2238+
goto default_label;
2239+
}
2240+
break;
2241+
case BPF_STORE_REL:
2242+
switch (BPF_SIZE(insn->code)) {
2243+
#define STORE_RELEASE(SIZEOP, SIZE) \
2244+
case BPF_##SIZEOP: \
2245+
smp_store_release( \
2246+
(SIZE *)(unsigned long)(DST + insn->off), (SIZE)SRC); \
2247+
break;
2248+
STORE_RELEASE(B, u8)
2249+
STORE_RELEASE(H, u16)
2250+
STORE_RELEASE(W, u32)
2251+
#ifdef CONFIG_64BIT
2252+
STORE_RELEASE(DW, u64)
2253+
#endif
2254+
#undef STORE_RELEASE
2255+
default:
2256+
goto default_label;
2257+
}
22032258
break;
22042259

22052260
default:

‎kernel/bpf/disasm.c‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,18 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
267267
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
268268
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
269269
insn->dst_reg, insn->off, insn->src_reg);
270+
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
271+
insn->imm == BPF_LOAD_ACQ) {
272+
verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n",
273+
insn->code, insn->dst_reg,
274+
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
275+
insn->src_reg, insn->off);
276+
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
277+
insn->imm == BPF_STORE_REL) {
278+
verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n",
279+
insn->code,
280+
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
281+
insn->dst_reg, insn->off, insn->src_reg);
270282
} else {
271283
verbose(cbs->private_data, "BUG_%02x\n", insn->code);
272284
}

‎kernel/bpf/verifier.c‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,13 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn)
579579
insn->imm == BPF_CMPXCHG;
580580
}
581581

582+
static bool is_atomic_load_insn(const struct bpf_insn *insn)
583+
{
584+
return BPF_CLASS(insn->code) == BPF_STX &&
585+
BPF_MODE(insn->code) == BPF_ATOMIC &&
586+
insn->imm == BPF_LOAD_ACQ;
587+
}
588+
582589
static int __get_spi(s32 off)
583590
{
584591
return (-off - 1) / BPF_REG_SIZE;
@@ -3567,7 +3574,7 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
35673574
}
35683575

35693576
if (class == BPF_STX) {
3570-
/* BPF_STX (including atomic variants) has multiple source
3577+
/* BPF_STX (including atomic variants) has one or more source
35713578
* operands, one of which is a ptr. Check whether the caller is
35723579
* asking about it.
35733580
*/
@@ -4181,7 +4188,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
41814188
* dreg still needs precision before this insn
41824189
*/
41834190
}
4184-
} else if (class == BPF_LDX) {
4191+
} else if (class == BPF_LDX || is_atomic_load_insn(insn)) {
41854192
if (!bt_is_reg_set(bt, dreg))
41864193
return 0;
41874194
bt_clear_reg(bt, dreg);
@@ -7766,6 +7773,32 @@ static int check_atomic_rmw(struct bpf_verifier_env *env,
77667773
return 0;
77677774
}
77687775

7776+
static int check_atomic_load(struct bpf_verifier_env *env,
7777+
struct bpf_insn *insn)
7778+
{
7779+
if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {
7780+
verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",
7781+
insn->src_reg,
7782+
reg_type_str(env, reg_state(env, insn->src_reg)->type));
7783+
return -EACCES;
7784+
}
7785+
7786+
return check_load_mem(env, insn, true, false, false, "atomic_load");
7787+
}
7788+
7789+
static int check_atomic_store(struct bpf_verifier_env *env,
7790+
struct bpf_insn *insn)
7791+
{
7792+
if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
7793+
verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
7794+
insn->dst_reg,
7795+
reg_type_str(env, reg_state(env, insn->dst_reg)->type));
7796+
return -EACCES;
7797+
}
7798+
7799+
return check_store_reg(env, insn, true);
7800+
}
7801+
77697802
static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
77707803
{
77717804
switch (insn->imm) {
@@ -7780,6 +7813,20 @@ static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
77807813
case BPF_XCHG:
77817814
case BPF_CMPXCHG:
77827815
return check_atomic_rmw(env, insn);
7816+
case BPF_LOAD_ACQ:
7817+
if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
7818+
verbose(env,
7819+
"64-bit load-acquires are only supported on 64-bit arches\n");
7820+
return -EOPNOTSUPP;
7821+
}
7822+
return check_atomic_load(env, insn);
7823+
case BPF_STORE_REL:
7824+
if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
7825+
verbose(env,
7826+
"64-bit store-releases are only supported on 64-bit arches\n");
7827+
return -EOPNOTSUPP;
7828+
}
7829+
return check_atomic_store(env, insn);
77837830
default:
77847831
verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",
77857832
insn->imm);
@@ -20592,7 +20639,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
2059220639
insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
2059320640
insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
2059420641
type = BPF_WRITE;
20595-
} else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
20642+
} else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) ||
20643+
insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) ||
20644+
insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
2059620645
insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) &&
2059720646
env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
2059820647
insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);

‎tools/include/uapi/linux/bpf.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@
5151
#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */
5252
#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */
5353

54+
#define BPF_LOAD_ACQ 0x100 /* load-acquire */
55+
#define BPF_STORE_REL 0x110 /* store-release */
56+
5457
enum bpf_cond_pseudo_jmp {
5558
BPF_MAY_GOTO = 0,
5659
};

0 commit comments

Comments
 (0)