x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit

JIRA: https://issues.redhat.com/browse/RHEL-88224

commit 318e8c3
Author: Patrick Bellasi <derkling@google.com>
Date:   Wed, 5 Feb 2025 14:04:41 +0000

    x86/cpu/kvm: SRSO: Fix possible missing IBPB on VM-Exit

    In [1] the meaning of the synthetic IBPB flags has been redefined for a
    better separation of concerns:
     - ENTRY_IBPB     -- issue IBPB on entry only
     - IBPB_ON_VMEXIT -- issue IBPB on VM-Exit only
    and the Retbleed mitigations have been updated to match this new
    semantics.

    Commit [2] was merged shortly before [1], and their interaction was not
    handled properly. This resulted in IBPB not being triggered on VM-Exit
    in all SRSO mitigation configs requesting an IBPB there.

    Specifically, an IBPB on VM-Exit is triggered only when
    X86_FEATURE_IBPB_ON_VMEXIT is set. However:

     - X86_FEATURE_IBPB_ON_VMEXIT is not set for "spec_rstack_overflow=ibpb",
       because before [1] having X86_FEATURE_ENTRY_IBPB was enough. Hence,
       an IBPB is triggered on entry but the expected IBPB on VM-exit is
       not.

     - X86_FEATURE_IBPB_ON_VMEXIT is not set also when
       "spec_rstack_overflow=ibpb-vmexit" if X86_FEATURE_ENTRY_IBPB is
       already set.

       That's because before [1] this was effectively redundant. Hence, e.g.
       a "retbleed=ibpb spec_rstack_overflow=bpb-vmexit" config mistakenly
       reports the machine still vulnerable to SRSO, despite an IBPB being
       triggered both on entry and VM-Exit, because of the Retbleed selected
       mitigation config.

     - UNTRAIN_RET_VM won't still actually do anything unless
       CONFIG_MITIGATION_IBPB_ENTRY is set.

    For "spec_rstack_overflow=ibpb", enable IBPB on both entry and VM-Exit
    and clear X86_FEATURE_RSB_VMEXIT which is made superfluous by
    X86_FEATURE_IBPB_ON_VMEXIT. This effectively makes this mitigation
    option similar to the one for 'retbleed=ibpb', thus re-order the code
    for the RETBLEED_MITIGATION_IBPB option to be less confusing by having
    all features enabling before the disabling of the not needed ones.

    For "spec_rstack_overflow=ibpb-vmexit", guard this mitigation setting
    with CONFIG_MITIGATION_IBPB_ENTRY to ensure UNTRAIN_RET_VM sequence is
    effectively compiled in. Drop instead the CONFIG_MITIGATION_SRSO guard,
    since none of the SRSO compile cruft is required in this configuration.
    Also, check only that the required microcode is present to effectively
    enabled the IBPB on VM-Exit.

    Finally, update the KConfig description for CONFIG_MITIGATION_IBPB_ENTRY
    to list also all SRSO config settings enabled by this guard.

    Fixes: 864bcaa ("x86/cpu/kvm: Provide UNTRAIN_RET_VM") [1]
    Fixes: d893832 ("x86/srso: Add IBPB on VMEXIT") [2]
    Reported-by: Yosry Ahmed <yosryahmed@google.com>
    Signed-off-by: Patrick Bellasi <derkling@google.com>
    Reviewed-by: Borislav Petkov (AMD) <bp@alien8.de>
    Cc: stable@kernel.org
    Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Signed-off-by: Waiman Long <longman@redhat.com>

Author: Waiman-Long

arch/x86/Kconfig

@@ -2560,7 +2560,8 @@ config MITIGATION_IBPB_ENTRY
 	depends on CPU_SUP_AMD && X86_64
 	default y
 	help
-	  Compile the kernel with support for the retbleed=ibpb mitigation.
+	  Compile the kernel with support for the retbleed=ibpb and
+	  spec_rstack_overflow={ibpb,ibpb-vmexit} mitigations.
 
 config MITIGATION_IBRS_ENTRY
 	bool "Enable IBRS on kernel entry"

arch/x86/kernel/cpu/bugs.c

@@ -1122,6 +1122,8 @@ static void __init retbleed_select_mitigation(void)
 
 	case RETBLEED_MITIGATION_IBPB:
 		setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
+		setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
+		mitigate_smt = true;
 
 		/*
 		 * IBPB on entry already obviates the need for
@@ -1131,9 +1133,6 @@ static void __init retbleed_select_mitigation(void)
 		setup_clear_cpu_cap(X86_FEATURE_UNRET);
 		setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
 
-		setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
-		mitigate_smt = true;
-
 		/*
 		 * There is no need for RSB filling: entry_ibpb() ensures
 		 * all predictions, including the RSB, are invalidated,
@@ -2731,6 +2730,7 @@ static void __init srso_select_mitigation(void)
 		if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
 			if (has_microcode) {
 				setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
+				setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
 				srso_mitigation = SRSO_MITIGATION_IBPB;
 
 				/*
@@ -2740,6 +2740,13 @@ static void __init srso_select_mitigation(void)
 				 */
 				setup_clear_cpu_cap(X86_FEATURE_UNRET);
 				setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
+
+				/*
+				 * There is no need for RSB filling: entry_ibpb() ensures
+				 * all predictions, including the RSB, are invalidated,
+				 * regardless of IBPB implementation.
+				 */
+				setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
 			}
 		} else {
 			pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
@@ -2748,8 +2755,8 @@ static void __init srso_select_mitigation(void)
 
 ibpb_on_vmexit:
 	case SRSO_CMD_IBPB_ON_VMEXIT:
-		if (IS_ENABLED(CONFIG_MITIGATION_SRSO)) {
-			if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) {
+		if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
+			if (has_microcode) {
 				setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
 				srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
 
@@ -2761,8 +2768,8 @@ static void __init srso_select_mitigation(void)
 				setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
 			}
 		} else {
-			pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n");
-                }
+			pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
+		}
 		break;
 	default:
 		break;