Skip to content

Commit 1a92370

Browse files
Waiman-LongWaiman-Long
committed
x86/retpoline: Ensure default return thunk isn't used at runtime
JIRA: https://issues.redhat.com/browse/RHEL-71467 Conflicts: 1) A merge conflict in the arch/x86/entry/entry.S hunk due to upstream merge conflict with resolution based on merge commit 685d982 ("Merge tag 'x86-core-2024-03-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip"). 2) A merge conflict in the arch/x86/entry/thunk_32.S hunk due to missing upstream commit 94ea9c0 ("x86/headers: Replace #include <asm/export.h> with #include <linux/export.h>"). commit 4461438 Author: Josh Poimboeuf <jpoimboe@kernel.org> Date: Wed, 3 Jan 2024 19:36:26 +0100 x86/retpoline: Ensure default return thunk isn't used at runtime Make sure the default return thunk is not used after all return instructions have been patched by the alternatives because the default return thunk is insufficient when it comes to mitigating Retbleed or SRSO. Fix based on an earlier version by David Kaplan <david.kaplan@amd.com>. [ bp: Fix the compilation error of warn_thunk_thunk being an invisible symbol, hoist thunk macro into calling.h ] Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org> Co-developed-by: Borislav Petkov (AMD) <bp@alien8.de> Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de> Link: https://lore.kernel.org/r/20231010171020.462211-4-david.kaplan@amd.com Link: https://lore.kernel.org/r/20240104132446.GEZZaxnrIgIyat0pqf@fat_crate.local Signed-off-by: Waiman Long <longman@redhat.com>
1 parent de15ea2 commit 1a92370

File tree

7 files changed

+84
-68
lines changed

7 files changed

+84
-68
lines changed

‎arch/x86/entry/calling.h‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -426,3 +426,63 @@ For 32-bit we have the following conventions - kernel is built with
426426
.endm
427427

428428
#endif /* CONFIG_SMP */
429+
430+
#ifdef CONFIG_X86_64
431+
432+
/* rdi: arg1 ... normal C conventions. rax is saved/restored. */
433+
.macro THUNK name, func
434+
SYM_FUNC_START(\name)
435+
pushq %rbp
436+
movq %rsp, %rbp
437+
438+
pushq %rdi
439+
pushq %rsi
440+
pushq %rdx
441+
pushq %rcx
442+
pushq %rax
443+
pushq %r8
444+
pushq %r9
445+
pushq %r10
446+
pushq %r11
447+
448+
call \func
449+
450+
popq %r11
451+
popq %r10
452+
popq %r9
453+
popq %r8
454+
popq %rax
455+
popq %rcx
456+
popq %rdx
457+
popq %rsi
458+
popq %rdi
459+
popq %rbp
460+
RET
461+
SYM_FUNC_END(\name)
462+
_ASM_NOKPROBE(\name)
463+
.endm
464+
465+
#else /* CONFIG_X86_32 */
466+
467+
/* put return address in eax (arg1) */
468+
.macro THUNK name, func, put_ret_addr_in_eax=0
469+
SYM_CODE_START_NOALIGN(\name)
470+
pushl %eax
471+
pushl %ecx
472+
pushl %edx
473+
474+
.if \put_ret_addr_in_eax
475+
/* Place EIP in the arg1 */
476+
movl 3*4(%esp), %eax
477+
.endif
478+
479+
call \func
480+
popl %edx
481+
popl %ecx
482+
popl %eax
483+
RET
484+
_ASM_NOKPROBE(\name)
485+
SYM_CODE_END(\name)
486+
.endm
487+
488+
#endif

‎arch/x86/entry/entry.S‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
#include <asm/segment.h>
1111
#include <asm/cache.h>
1212

13+
#include "calling.h"
14+
1315
.pushsection .noinstr.text, "ax"
1416

1517
SYM_FUNC_START(entry_ibpb)
@@ -43,3 +45,4 @@ EXPORT_SYMBOL_GPL(mds_verw_sel);
4345

4446
.popsection
4547

48+
THUNK warn_thunk_thunk, __warn_thunk

‎arch/x86/entry/thunk_32.S‎

Lines changed: 8 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,15 @@
44
* Copyright 2008 by Steven Rostedt, Red Hat, Inc
55
* (inspired by Andi Kleen's thunk_64.S)
66
*/
7-
#include <linux/linkage.h>
8-
#include <asm/asm.h>
9-
#include <asm/export.h>
107

11-
/* put return address in eax (arg1) */
12-
.macro THUNK name, func, put_ret_addr_in_eax=0
13-
SYM_CODE_START_NOALIGN(\name)
14-
pushl %eax
15-
pushl %ecx
16-
pushl %edx
8+
#include <linux/linkage.h>
9+
#include <asm/asm.h>
10+
#include <asm/export.h>
1711

18-
.if \put_ret_addr_in_eax
19-
/* Place EIP in the arg1 */
20-
movl 3*4(%esp), %eax
21-
.endif
12+
#include "calling.h"
2213

23-
call \func
24-
popl %edx
25-
popl %ecx
26-
popl %eax
27-
RET
28-
_ASM_NOKPROBE(\name)
29-
SYM_CODE_END(\name)
30-
.endm
31-
32-
THUNK preempt_schedule_thunk, preempt_schedule
33-
THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace
34-
EXPORT_SYMBOL(preempt_schedule_thunk)
35-
EXPORT_SYMBOL(preempt_schedule_notrace_thunk)
14+
THUNK preempt_schedule_thunk, preempt_schedule
15+
THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace
16+
EXPORT_SYMBOL(preempt_schedule_thunk)
17+
EXPORT_SYMBOL(preempt_schedule_notrace_thunk)
3618

‎arch/x86/entry/thunk_64.S‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -9,39 +9,6 @@
99
#include <asm/asm.h>
1010
#include <asm/export.h>
1111

12-
/* rdi: arg1 ... normal C conventions. rax is saved/restored. */
13-
.macro THUNK name, func
14-
SYM_FUNC_START(\name)
15-
pushq %rbp
16-
movq %rsp, %rbp
17-
18-
pushq %rdi
19-
pushq %rsi
20-
pushq %rdx
21-
pushq %rcx
22-
pushq %rax
23-
pushq %r8
24-
pushq %r9
25-
pushq %r10
26-
pushq %r11
27-
28-
call \func
29-
30-
popq %r11
31-
popq %r10
32-
popq %r9
33-
popq %r8
34-
popq %rax
35-
popq %rcx
36-
popq %rdx
37-
popq %rsi
38-
popq %rdi
39-
popq %rbp
40-
RET
41-
SYM_FUNC_END(\name)
42-
_ASM_NOKPROBE(\name)
43-
.endm
44-
4512
THUNK preempt_schedule_thunk, preempt_schedule
4613
THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace
4714
EXPORT_SYMBOL(preempt_schedule_thunk)

‎arch/x86/include/asm/nospec-branch.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,8 @@ extern void clear_bhb_loop(void);
396396

397397
extern void (*x86_return_thunk)(void);
398398

399+
extern void __warn_thunk(void);
400+
399401
#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
400402
extern void call_depth_return_thunk(void);
401403

‎arch/x86/kernel/cpu/bugs.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3102,3 +3102,8 @@ ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attrib
31023102
return cpu_show_common(dev, attr, buf, X86_BUG_RFDS);
31033103
}
31043104
#endif
3105+
3106+
void __warn_thunk(void)
3107+
{
3108+
WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n");
3109+
}

‎arch/x86/lib/retpoline.S‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -374,19 +374,16 @@ SYM_FUNC_END(call_depth_return_thunk)
374374
* 'JMP __x86_return_thunk' sites are changed to something else by
375375
* apply_returns().
376376
*
377-
* This should be converted eventually to call a warning function which
378-
* should scream loudly when the default return thunk is called after
379-
* alternatives have been applied.
380-
*
381-
* That warning function cannot BUG() because the bug splat cannot be
382-
* displayed in all possible configurations, leading to users not really
383-
* knowing why the machine froze.
377+
* The ALTERNATIVE below adds a really loud warning to catch the case
378+
* where the insufficient default return thunk ends up getting used for
379+
* whatever reason like miscompilation or failure of
380+
* objtool/alternatives/etc to patch all the return sites.
384381
*/
385382
SYM_CODE_START(__x86_return_thunk)
386383
UNWIND_HINT_FUNC
387384
ANNOTATE_NOENDBR
388-
ANNOTATE_UNRET_SAFE
389-
ret
385+
ALTERNATIVE __stringify(ANNOTATE_UNRET_SAFE; ret), \
386+
"jmp warn_thunk_thunk", X86_FEATURE_ALWAYS
390387
int3
391388
SYM_CODE_END(__x86_return_thunk)
392389
EXPORT_SYMBOL(__x86_return_thunk)

0 commit comments

Comments
 (0)