For each vCPU - store a linked list of all shadow physical id entries which address it.
- Update those entries when this vCPU is scheduled in/out
- update this list, when physid tables are modified by other means (guest write and/or table sync)
To avoid races vs vcpu schedule, use a spinlock.
Signed-off-by: Maxim Levitsky mlevitsk@redhat.com --- arch/x86/kvm/svm/avic.c | 113 +++++++++++++++++++++++++++++++++++++--- arch/x86/kvm/svm/svm.c | 7 +++ arch/x86/kvm/svm/svm.h | 10 ++++ 3 files changed, 122 insertions(+), 8 deletions(-)
diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index f462b7e48e3ca..34da9fabd5194 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -67,8 +67,12 @@ static void avic_physid_shadow_entry_set_vcpu(struct kvm *kvm, struct avic_physid_entry_descr *e = &t->entries[n]; u64 sentry = READ_ONCE(*e->sentry); u64 old_sentry = sentry; + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); struct kvm_vcpu *new_vcpu = NULL; int l0_apicid = -1; + unsigned long flags; + + raw_spin_lock_irqsave(&kvm_svm->avic.table_entries_lock, flags);
WARN_ON(!test_bit(n, t->valid_entires));
@@ -79,6 +83,9 @@ static void avic_physid_shadow_entry_set_vcpu(struct kvm *kvm, new_vcpu = avic_vcpu_by_l1_apicid(kvm, new_l1_apicid);
if (new_vcpu) + list_add_tail(&e->link, &to_svm(new_vcpu)->nested.physid_ref_entries); + + if (new_vcpu && to_svm(new_vcpu)->nested_avic_active) l0_apicid = kvm_cpu_get_apicid(new_vcpu->cpu);
physid_entry_set_apicid(&sentry, l0_apicid); @@ -87,6 +94,8 @@ static void avic_physid_shadow_entry_set_vcpu(struct kvm *kvm,
if (sentry != old_sentry) WRITE_ONCE(*e->sentry, sentry); + + raw_spin_unlock_irqrestore(&kvm_svm->avic.table_entries_lock, flags); }
static void avic_physid_shadow_entry_create(struct kvm *kvm, @@ -131,7 +140,11 @@ static void avic_physid_shadow_entry_remove(struct kvm *kvm, int n) { struct avic_physid_entry_descr *e = &t->entries[n]; + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); hpa_t backing_page_hpa; + unsigned long flags; + + raw_spin_lock_irqsave(&kvm_svm->avic.table_entries_lock, flags);
if (!test_and_clear_bit(n, t->valid_entires)) WARN_ON(1); @@ -147,8 +160,49 @@ static void avic_physid_shadow_entry_remove(struct kvm *kvm,
e->gentry = 0; *e->sentry = 0; + + raw_spin_unlock_irqrestore(&kvm_svm->avic.table_entries_lock, flags); }
+static void avic_update_peer_physid_entries(struct kvm_vcpu *vcpu, int cpu) +{ + /* + * Update all shadow physid tables which contain entries + * which reference this vCPU with its new physical location + */ + struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); + struct vcpu_svm *vcpu_svm = to_svm(vcpu); + struct avic_physid_entry_descr *e; + int updated_nentries = 0; + int l0_apicid = -1; + unsigned long flags; + bool new_active = cpu != -1; + + if (cpu != -1) + l0_apicid = kvm_cpu_get_apicid(cpu); + + raw_spin_lock_irqsave(&kvm_svm->avic.table_entries_lock, flags); + + list_for_each_entry(e, &vcpu_svm->nested.physid_ref_entries, link) { + u64 sentry = READ_ONCE(*e->sentry); + u64 old_sentry = sentry; + + physid_entry_set_apicid(&sentry, l0_apicid); + + if (sentry != old_sentry) { + updated_nentries++; + WRITE_ONCE(*e->sentry, sentry); + } + } + + if (updated_nentries) + trace_kvm_avic_physid_update_vcpu_host(vcpu->vcpu_id, + l0_apicid, updated_nentries); + + vcpu_svm->nested_avic_active = new_active; + + raw_spin_unlock_irqrestore(&kvm_svm->avic.table_entries_lock, flags); +}
static bool avic_physid_shadow_table_setup_write_tracking(struct kvm *kvm, @@ -603,6 +657,7 @@ int avic_vm_init(struct kvm *kvm) hash_add(svm_vm_data_hash, &avic->hnode, avic->vm_id); spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
+ raw_spin_lock_init(&avic->table_entries_lock); mutex_init(&avic->tables_lock); INIT_LIST_HEAD(&avic->physid_tables);
@@ -1428,9 +1483,51 @@ static void avic_vcpu_load(struct kvm_vcpu *vcpu) static void avic_vcpu_put(struct kvm_vcpu *vcpu) { preempt_disable(); - __avic_vcpu_put(vcpu); + preempt_enable(); +} +
+void __nested_avic_load(struct kvm_vcpu *vcpu, int cpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + lockdep_assert_preemption_disabled(); + + /* + * For the same reason as in __avic_vcpu_load there is no + * need to load nested AVIC when this vCPU is blocking + */ + if (kvm_vcpu_is_blocking(vcpu)) + return; + + if (svm->nested.initialized) + avic_update_peer_physid_entries(vcpu, cpu); +} + +void __nested_avic_put(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + lockdep_assert_preemption_disabled(); + + if (svm->nested.initialized) + avic_update_peer_physid_entries(vcpu, -1); +} + +void nested_avic_load(struct kvm_vcpu *vcpu) +{ + int cpu = get_cpu(); + + WARN_ON(cpu != vcpu->cpu); + __nested_avic_load(vcpu, cpu); + put_cpu(); +} + +void nested_avic_put(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + __nested_avic_put(vcpu); preempt_enable(); }
@@ -1468,9 +1565,6 @@ void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
void avic_vcpu_blocking(struct kvm_vcpu *vcpu) { - if (!kvm_vcpu_apicv_active(vcpu)) - return; - /* * Unload the AVIC when the vCPU is about to block, _before_ * the vCPU actually blocks. @@ -1484,13 +1578,16 @@ void avic_vcpu_blocking(struct kvm_vcpu *vcpu) * IRR and reading IsRunning; the lack of this barrier might be * the cause of errata #1235). */ - avic_vcpu_put(vcpu); + if (kvm_vcpu_apicv_active(vcpu)) + avic_vcpu_put(vcpu); + + nested_avic_put(vcpu); }
void avic_vcpu_unblocking(struct kvm_vcpu *vcpu) { - if (!kvm_vcpu_apicv_active(vcpu)) - return; + if (kvm_vcpu_apicv_active(vcpu)) + avic_vcpu_load(vcpu);
- avic_vcpu_load(vcpu); + nested_avic_load(vcpu); } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 75b4f3ac8b1a0..76fbee2c8c5d7 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1302,6 +1302,8 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu)
svm->guest_state_loaded = false;
+ INIT_LIST_HEAD(&svm->nested.physid_ref_entries); + return 0;
error_free_vmsa_page: @@ -1391,8 +1393,11 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) sd->current_vmcb = svm->vmcb; indirect_branch_prediction_barrier(); } + if (kvm_vcpu_apicv_active(vcpu)) __avic_vcpu_load(vcpu, cpu); + + __nested_avic_load(vcpu, cpu); }
static void svm_vcpu_put(struct kvm_vcpu *vcpu) @@ -1400,6 +1405,8 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) if (kvm_vcpu_apicv_active(vcpu)) __avic_vcpu_put(vcpu);
+ __nested_avic_put(vcpu); + svm_prepare_host_switch(vcpu);
++vcpu->stat.host_state_reload; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index fc15e1f938793..401449dbce65d 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -99,6 +99,7 @@ struct kvm_svm_avic { struct page *physical_id_table_page; struct hlist_node hnode;
+ raw_spinlock_t table_entries_lock; struct mutex tables_lock;
/* List of all shadow tables */ @@ -244,6 +245,9 @@ struct svm_nested_state { * on its side. */ bool force_msr_bitmap_recalc; + + /* All AVIC shadow PID table entry descriptors that reference this vCPU */ + struct list_head physid_ref_entries; };
struct vcpu_sev_es_state { @@ -311,6 +315,7 @@ struct vcpu_svm { u32 dfr_reg; struct page *avic_backing_page; u64 *avic_physical_id_cache; + bool nested_avic_active;
/* * Per-vcpu list of struct amd_svm_iommu_ir: @@ -678,6 +683,11 @@ int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu); int avic_init_vcpu(struct vcpu_svm *svm); void __avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void __avic_vcpu_put(struct kvm_vcpu *vcpu); +void __nested_avic_load(struct kvm_vcpu *vcpu, int cpu); +void __nested_avic_put(struct kvm_vcpu *vcpu); +void nested_avic_load(struct kvm_vcpu *vcpu); +void nested_avic_put(struct kvm_vcpu *vcpu); + void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu); void avic_set_virtual_apic_mode(struct kvm_vcpu *vcpu); void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu);