Skip to content

Commit 90d862f

Browse files
hbathinimpe
hbathini
authored and
mpe
committed
powerpc/bpf: use bpf_jit_binary_pack_[alloc|finalize|free]
Use bpf_jit_binary_pack_alloc in powerpc jit. The jit engine first writes the program to the rw buffer. When the jit is done, the program is copied to the final location with bpf_jit_binary_pack_finalize. With multiple jit_subprogs, bpf_jit_free is called on some subprograms that haven't got bpf_jit_binary_pack_finalize() yet. Implement custom bpf_jit_free() like in commit 1d5f82d ("bpf, x86: fix freeing of not-finalized bpf_prog_pack") to call bpf_jit_binary_pack_finalize(), if necessary. As bpf_flush_icache() is not needed anymore, remove it. Signed-off-by: Hari Bathini <hbathini@linux.ibm.com> Acked-by: Song Liu <song@kernel.org> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://msgid.link/20231020141358.643575-6-hbathini@linux.ibm.com
1 parent de04e40 commit 90d862f

File tree

4 files changed

+96
-51
lines changed

4 files changed

+96
-51
lines changed

‎arch/powerpc/net/bpf_jit.h‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,6 @@
3636
EMIT(PPC_RAW_BRANCH(offset)); \
3737
} while (0)
3838

39-
/* bl (unconditional 'branch' with link) */
40-
#define PPC_BL(dest) EMIT(PPC_RAW_BL((dest) - (unsigned long)(image + ctx->idx)))
41-
4239
/* "cond" here covers BO:BI fields. */
4340
#define PPC_BCC_SHORT(cond, dest) \
4441
do { \
@@ -147,12 +144,6 @@ struct codegen_context {
147144
#define BPF_FIXUP_LEN 2 /* Two instructions => 8 bytes */
148145
#endif
149146

150-
static inline void bpf_flush_icache(void *start, void *end)
151-
{
152-
smp_wmb(); /* smp write barrier */
153-
flush_icache_range((unsigned long)start, (unsigned long)end);
154-
}
155-
156147
static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
157148
{
158149
return ctx->seen & (1 << (31 - i));
@@ -169,16 +160,17 @@ static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i)
169160
}
170161

171162
void bpf_jit_init_reg_mapping(struct codegen_context *ctx);
172-
int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func);
173-
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
163+
int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func);
164+
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx,
174165
u32 *addrs, int pass, bool extra_pass);
175166
void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx);
176167
void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx);
177168
void bpf_jit_realloc_regs(struct codegen_context *ctx);
178169
int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr);
179170

180-
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
181-
int insn_idx, int jmp_off, int dst_reg);
171+
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
172+
struct codegen_context *ctx, int insn_idx,
173+
int jmp_off, int dst_reg);
182174

183175
#endif
184176

‎arch/powerpc/net/bpf_jit_comp.c‎

Lines changed: 79 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,12 @@ int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg,
4444
}
4545

4646
struct powerpc_jit_data {
47-
struct bpf_binary_header *header;
47+
/* address of rw header */
48+
struct bpf_binary_header *hdr;
49+
/* address of ro final header */
50+
struct bpf_binary_header *fhdr;
4851
u32 *addrs;
49-
u8 *image;
52+
u8 *fimage;
5053
u32 proglen;
5154
struct codegen_context ctx;
5255
};
@@ -67,11 +70,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
6770
struct codegen_context cgctx;
6871
int pass;
6972
int flen;
70-
struct bpf_binary_header *bpf_hdr;
73+
struct bpf_binary_header *fhdr = NULL;
74+
struct bpf_binary_header *hdr = NULL;
7175
struct bpf_prog *org_fp = fp;
7276
struct bpf_prog *tmp_fp;
7377
bool bpf_blinded = false;
7478
bool extra_pass = false;
79+
u8 *fimage = NULL;
80+
u32 *fcode_base;
7581
u32 extable_len;
7682
u32 fixup_len;
7783

@@ -101,9 +107,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
101107
addrs = jit_data->addrs;
102108
if (addrs) {
103109
cgctx = jit_data->ctx;
104-
image = jit_data->image;
105-
bpf_hdr = jit_data->header;
110+
/*
111+
* JIT compiled to a writable location (image/code_base) first.
112+
* It is then moved to the readonly final location (fimage/fcode_base)
113+
* using instruction patching.
114+
*/
115+
fimage = jit_data->fimage;
116+
fhdr = jit_data->fhdr;
106117
proglen = jit_data->proglen;
118+
hdr = jit_data->hdr;
119+
image = (void *)hdr + ((void *)fimage - (void *)fhdr);
107120
extra_pass = true;
108121
/* During extra pass, ensure index is reset before repopulating extable entries */
109122
cgctx.exentry_idx = 0;
@@ -123,7 +136,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
123136
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
124137

125138
/* Scouting faux-generate pass 0 */
126-
if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) {
139+
if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
127140
/* We hit something illegal or unsupported. */
128141
fp = org_fp;
129142
goto out_addrs;
@@ -138,7 +151,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
138151
*/
139152
if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
140153
cgctx.idx = 0;
141-
if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) {
154+
if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
142155
fp = org_fp;
143156
goto out_addrs;
144157
}
@@ -160,26 +173,30 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
160173
proglen = cgctx.idx * 4;
161174
alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
162175

163-
bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
164-
if (!bpf_hdr) {
176+
fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
177+
bpf_jit_fill_ill_insns);
178+
if (!fhdr) {
165179
fp = org_fp;
166180
goto out_addrs;
167181
}
168182

169183
if (extable_len)
170-
fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len;
184+
fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
171185

172186
skip_init_ctx:
173187
code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
188+
fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
174189

175190
/* Code generation passes 1-2 */
176191
for (pass = 1; pass < 3; pass++) {
177192
/* Now build the prologue, body code & epilogue for real. */
178193
cgctx.idx = 0;
179194
cgctx.alt_exit_addr = 0;
180195
bpf_jit_build_prologue(code_base, &cgctx);
181-
if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass, extra_pass)) {
182-
bpf_jit_binary_free(bpf_hdr);
196+
if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
197+
extra_pass)) {
198+
bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
199+
bpf_jit_binary_pack_free(fhdr, hdr);
183200
fp = org_fp;
184201
goto out_addrs;
185202
}
@@ -199,17 +216,19 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
199216

200217
#ifdef CONFIG_PPC64_ELF_ABI_V1
201218
/* Function descriptor nastiness: Address + TOC */
202-
((u64 *)image)[0] = (u64)code_base;
219+
((u64 *)image)[0] = (u64)fcode_base;
203220
((u64 *)image)[1] = local_paca->kernel_toc;
204221
#endif
205222

206-
fp->bpf_func = (void *)image;
223+
fp->bpf_func = (void *)fimage;
207224
fp->jited = 1;
208225
fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
209226

210-
bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size);
211227
if (!fp->is_func || extra_pass) {
212-
bpf_jit_binary_lock_ro(bpf_hdr);
228+
if (bpf_jit_binary_pack_finalize(fp, fhdr, hdr)) {
229+
fp = org_fp;
230+
goto out_addrs;
231+
}
213232
bpf_prog_fill_jited_linfo(fp, addrs);
214233
out_addrs:
215234
kfree(addrs);
@@ -219,8 +238,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
219238
jit_data->addrs = addrs;
220239
jit_data->ctx = cgctx;
221240
jit_data->proglen = proglen;
222-
jit_data->image = image;
223-
jit_data->header = bpf_hdr;
241+
jit_data->fimage = fimage;
242+
jit_data->fhdr = fhdr;
243+
jit_data->hdr = hdr;
224244
}
225245

226246
out:
@@ -234,12 +254,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
234254
* The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
235255
* this function, as this only applies to BPF_PROBE_MEM, for now.
236256
*/
237-
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
238-
int insn_idx, int jmp_off, int dst_reg)
257+
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
258+
struct codegen_context *ctx, int insn_idx, int jmp_off,
259+
int dst_reg)
239260
{
240261
off_t offset;
241262
unsigned long pc;
242-
struct exception_table_entry *ex;
263+
struct exception_table_entry *ex, *ex_entry;
243264
u32 *fixup;
244265

245266
/* Populate extable entries only in the last pass */
@@ -250,9 +271,16 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct code
250271
WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
251272
return -EINVAL;
252273

274+
/*
275+
* Program is first written to image before copying to the
276+
* final location (fimage). Accordingly, update in the image first.
277+
* As all offsets used are relative, copying as is to the
278+
* final location should be alright.
279+
*/
253280
pc = (unsigned long)&image[insn_idx];
281+
ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
254282

255-
fixup = (void *)fp->aux->extable -
283+
fixup = (void *)ex -
256284
(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
257285
(ctx->exentry_idx * BPF_FIXUP_LEN * 4);
258286

@@ -263,17 +291,17 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct code
263291
fixup[BPF_FIXUP_LEN - 1] =
264292
PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
265293

266-
ex = &fp->aux->extable[ctx->exentry_idx];
294+
ex_entry = &ex[ctx->exentry_idx];
267295

268-
offset = pc - (long)&ex->insn;
296+
offset = pc - (long)&ex_entry->insn;
269297
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
270298
return -ERANGE;
271-
ex->insn = offset;
299+
ex_entry->insn = offset;
272300

273-
offset = (long)fixup - (long)&ex->fixup;
301+
offset = (long)fixup - (long)&ex_entry->fixup;
274302
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
275303
return -ERANGE;
276-
ex->fixup = offset;
304+
ex_entry->fixup = offset;
277305

278306
ctx->exentry_idx++;
279307
return 0;
@@ -307,3 +335,27 @@ int bpf_arch_text_invalidate(void *dst, size_t len)
307335

308336
return ret;
309337
}
338+
339+
void bpf_jit_free(struct bpf_prog *fp)
340+
{
341+
if (fp->jited) {
342+
struct powerpc_jit_data *jit_data = fp->aux->jit_data;
343+
struct bpf_binary_header *hdr;
344+
345+
/*
346+
* If we fail the final pass of JIT (from jit_subprogs),
347+
* the program may not be finalized yet. Call finalize here
348+
* before freeing it.
349+
*/
350+
if (jit_data) {
351+
bpf_jit_binary_pack_finalize(fp, jit_data->fhdr, jit_data->hdr);
352+
kvfree(jit_data->addrs);
353+
kfree(jit_data);
354+
}
355+
hdr = bpf_jit_binary_pack_hdr(fp);
356+
bpf_jit_binary_pack_free(hdr, NULL);
357+
WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
358+
}
359+
360+
bpf_prog_unlock_free(fp);
361+
}

‎arch/powerpc/net/bpf_jit_comp32.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -200,12 +200,13 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
200200
EMIT(PPC_RAW_BLR());
201201
}
202202

203-
int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
203+
/* Relative offset needs to be calculated based on final image location */
204+
int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func)
204205
{
205-
s32 rel = (s32)func - (s32)(image + ctx->idx);
206+
s32 rel = (s32)func - (s32)(fimage + ctx->idx);
206207

207208
if (image && rel < 0x2000000 && rel >= -0x2000000) {
208-
PPC_BL(func);
209+
EMIT(PPC_RAW_BL(rel));
209210
} else {
210211
/* Load function address into r0 */
211212
EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
@@ -278,7 +279,7 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o
278279
}
279280

280281
/* Assemble the body code between the prologue & epilogue */
281-
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
282+
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx,
282283
u32 *addrs, int pass, bool extra_pass)
283284
{
284285
const struct bpf_insn *insn = fp->insnsi;
@@ -997,7 +998,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
997998
jmp_off += 4;
998999
}
9991000

1000-
ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx,
1001+
ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, insn_idx,
10011002
jmp_off, dst_reg);
10021003
if (ret)
10031004
return ret;
@@ -1053,7 +1054,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
10531054
EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12));
10541055
}
10551056

1056-
ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1057+
ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr);
10571058
if (ret)
10581059
return ret;
10591060

‎arch/powerpc/net/bpf_jit_comp64.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ static int bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, u
240240
return 0;
241241
}
242242

243-
int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
243+
int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func)
244244
{
245245
unsigned int i, ctx_idx = ctx->idx;
246246

@@ -361,7 +361,7 @@ asm (
361361
);
362362

363363
/* Assemble the body code between the prologue & epilogue */
364-
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
364+
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx,
365365
u32 *addrs, int pass, bool extra_pass)
366366
{
367367
enum stf_barrier_type stf_barrier = stf_barrier_type_get();
@@ -940,8 +940,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
940940
addrs[++i] = ctx->idx * 4;
941941

942942
if (BPF_MODE(code) == BPF_PROBE_MEM) {
943-
ret = bpf_add_extable_entry(fp, image, pass, ctx, ctx->idx - 1,
944-
4, dst_reg);
943+
ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,
944+
ctx->idx - 1, 4, dst_reg);
945945
if (ret)
946946
return ret;
947947
}
@@ -995,7 +995,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
995995
if (func_addr_fixed)
996996
ret = bpf_jit_emit_func_call_hlp(image, ctx, func_addr);
997997
else
998-
ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
998+
ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr);
999999

10001000
if (ret)
10011001
return ret;

0 commit comments

Comments
 (0)