Skip to content

Commit 9fb13ba

Browse files
sean-jcsean-jc
committed
KVM: x86/mmu: Refactor low level rmap helpers to prep for walking w/o mmu_lock
Refactor the pte_list and rmap code to always read and write rmap_head->val exactly once, e.g. by collecting changes in a local variable and then propagating those changes back to rmap_head->val as appropriate. This will allow implementing a per-rmap rwlock (of sorts) by adding a LOCKED bit into the rmap value alongside the MANY bit. Signed-off-by: James Houghton <jthoughton@google.com> Acked-by: Yu Zhao <yuzhao@google.com> Reviewed-by: James Houghton <jthoughton@google.com> Link: https://lore.kernel.org/r/20250204004038.1680123-9-jthoughton@google.com Signed-off-by: Sean Christopherson <seanjc@google.com>
1 parent 8c403cf commit 9fb13ba

File tree

1 file changed

+50
-33
lines changed

1 file changed

+50
-33
lines changed

‎arch/x86/kvm/mmu/mmu.c‎

Lines changed: 50 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -864,21 +864,24 @@ static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu
864864
static int pte_list_add(struct kvm_mmu_memory_cache *cache, u64 *spte,
865865
struct kvm_rmap_head *rmap_head)
866866
{
867+
unsigned long old_val, new_val;
867868
struct pte_list_desc *desc;
868869
int count = 0;
869870

870-
if (!rmap_head->val) {
871-
rmap_head->val = (unsigned long)spte;
872-
} else if (!(rmap_head->val & KVM_RMAP_MANY)) {
871+
old_val = rmap_head->val;
872+
873+
if (!old_val) {
874+
new_val = (unsigned long)spte;
875+
} else if (!(old_val & KVM_RMAP_MANY)) {
873876
desc = kvm_mmu_memory_cache_alloc(cache);
874-
desc->sptes[0] = (u64 *)rmap_head->val;
877+
desc->sptes[0] = (u64 *)old_val;
875878
desc->sptes[1] = spte;
876879
desc->spte_count = 2;
877880
desc->tail_count = 0;
878-
rmap_head->val = (unsigned long)desc | KVM_RMAP_MANY;
881+
new_val = (unsigned long)desc | KVM_RMAP_MANY;
879882
++count;
880883
} else {
881-
desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);
884+
desc = (struct pte_list_desc *)(old_val & ~KVM_RMAP_MANY);
882885
count = desc->tail_count + desc->spte_count;
883886

884887
/*
@@ -887,21 +890,25 @@ static int pte_list_add(struct kvm_mmu_memory_cache *cache, u64 *spte,
887890
*/
888891
if (desc->spte_count == PTE_LIST_EXT) {
889892
desc = kvm_mmu_memory_cache_alloc(cache);
890-
desc->more = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);
893+
desc->more = (struct pte_list_desc *)(old_val & ~KVM_RMAP_MANY);
891894
desc->spte_count = 0;
892895
desc->tail_count = count;
893-
rmap_head->val = (unsigned long)desc | KVM_RMAP_MANY;
896+
new_val = (unsigned long)desc | KVM_RMAP_MANY;
897+
} else {
898+
new_val = old_val;
894899
}
895900
desc->sptes[desc->spte_count++] = spte;
896901
}
902+
903+
rmap_head->val = new_val;
904+
897905
return count;
898906
}
899907

900-
static void pte_list_desc_remove_entry(struct kvm *kvm,
901-
struct kvm_rmap_head *rmap_head,
908+
static void pte_list_desc_remove_entry(struct kvm *kvm, unsigned long *rmap_val,
902909
struct pte_list_desc *desc, int i)
903910
{
904-
struct pte_list_desc *head_desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);
911+
struct pte_list_desc *head_desc = (struct pte_list_desc *)(*rmap_val & ~KVM_RMAP_MANY);
905912
int j = head_desc->spte_count - 1;
906913

907914
/*
@@ -928,41 +935,46 @@ static void pte_list_desc_remove_entry(struct kvm *kvm,
928935
* head at the next descriptor, i.e. the new head.
929936
*/
930937
if (!head_desc->more)
931-
rmap_head->val = 0;
938+
*rmap_val = 0;
932939
else
933-
rmap_head->val = (unsigned long)head_desc->more | KVM_RMAP_MANY;
940+
*rmap_val = (unsigned long)head_desc->more | KVM_RMAP_MANY;
934941
mmu_free_pte_list_desc(head_desc);
935942
}
936943

937944
static void pte_list_remove(struct kvm *kvm, u64 *spte,
938945
struct kvm_rmap_head *rmap_head)
939946
{
940947
struct pte_list_desc *desc;
948+
unsigned long rmap_val;
941949
int i;
942950

943-
if (KVM_BUG_ON_DATA_CORRUPTION(!rmap_head->val, kvm))
944-
return;
951+
rmap_val = rmap_head->val;
952+
if (KVM_BUG_ON_DATA_CORRUPTION(!rmap_val, kvm))
953+
goto out;
945954

946-
if (!(rmap_head->val & KVM_RMAP_MANY)) {
947-
if (KVM_BUG_ON_DATA_CORRUPTION((u64 *)rmap_head->val != spte, kvm))
948-
return;
955+
if (!(rmap_val & KVM_RMAP_MANY)) {
956+
if (KVM_BUG_ON_DATA_CORRUPTION((u64 *)rmap_val != spte, kvm))
957+
goto out;
949958

950-
rmap_head->val = 0;
959+
rmap_val = 0;
951960
} else {
952-
desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);
961+
desc = (struct pte_list_desc *)(rmap_val & ~KVM_RMAP_MANY);
953962
while (desc) {
954963
for (i = 0; i < desc->spte_count; ++i) {
955964
if (desc->sptes[i] == spte) {
956-
pte_list_desc_remove_entry(kvm, rmap_head,
965+
pte_list_desc_remove_entry(kvm, &rmap_val,
957966
desc, i);
958-
return;
967+
goto out;
959968
}
960969
}
961970
desc = desc->more;
962971
}
963972

964973
KVM_BUG_ON_DATA_CORRUPTION(true, kvm);
965974
}
975+
976+
out:
977+
rmap_head->val = rmap_val;
966978
}
967979

968980
static void kvm_zap_one_rmap_spte(struct kvm *kvm,
@@ -977,17 +989,19 @@ static bool kvm_zap_all_rmap_sptes(struct kvm *kvm,
977989
struct kvm_rmap_head *rmap_head)
978990
{
979991
struct pte_list_desc *desc, *next;
992+
unsigned long rmap_val;
980993
int i;
981994

982-
if (!rmap_head->val)
995+
rmap_val = rmap_head->val;
996+
if (!rmap_val)
983997
return false;
984998

985-
if (!(rmap_head->val & KVM_RMAP_MANY)) {
986-
mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);
999+
if (!(rmap_val & KVM_RMAP_MANY)) {
1000+
mmu_spte_clear_track_bits(kvm, (u64 *)rmap_val);
9871001
goto out;
9881002
}
9891003

990-
desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);
1004+
desc = (struct pte_list_desc *)(rmap_val & ~KVM_RMAP_MANY);
9911005

9921006
for (; desc; desc = next) {
9931007
for (i = 0; i < desc->spte_count; i++)
@@ -1003,14 +1017,15 @@ static bool kvm_zap_all_rmap_sptes(struct kvm *kvm,
10031017

10041018
unsigned int pte_list_count(struct kvm_rmap_head *rmap_head)
10051019
{
1020+
unsigned long rmap_val = rmap_head->val;
10061021
struct pte_list_desc *desc;
10071022

1008-
if (!rmap_head->val)
1023+
if (!rmap_val)
10091024
return 0;
1010-
else if (!(rmap_head->val & KVM_RMAP_MANY))
1025+
else if (!(rmap_val & KVM_RMAP_MANY))
10111026
return 1;
10121027

1013-
desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);
1028+
desc = (struct pte_list_desc *)(rmap_val & ~KVM_RMAP_MANY);
10141029
return desc->tail_count + desc->spte_count;
10151030
}
10161031

@@ -1053,6 +1068,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
10531068
*/
10541069
struct rmap_iterator {
10551070
/* private fields */
1071+
struct rmap_head *head;
10561072
struct pte_list_desc *desc; /* holds the sptep if not NULL */
10571073
int pos; /* index of the sptep */
10581074
};
@@ -1067,18 +1083,19 @@ struct rmap_iterator {
10671083
static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
10681084
struct rmap_iterator *iter)
10691085
{
1086+
unsigned long rmap_val = rmap_head->val;
10701087
u64 *sptep;
10711088

1072-
if (!rmap_head->val)
1089+
if (!rmap_val)
10731090
return NULL;
10741091

1075-
if (!(rmap_head->val & KVM_RMAP_MANY)) {
1092+
if (!(rmap_val & KVM_RMAP_MANY)) {
10761093
iter->desc = NULL;
1077-
sptep = (u64 *)rmap_head->val;
1094+
sptep = (u64 *)rmap_val;
10781095
goto out;
10791096
}
10801097

1081-
iter->desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);
1098+
iter->desc = (struct pte_list_desc *)(rmap_val & ~KVM_RMAP_MANY);
10821099
iter->pos = 0;
10831100
sptep = iter->desc->sptes[iter->pos];
10841101
out:

0 commit comments

Comments
 (0)