From: David Stevens stevensd@chromium.org
Avoid converting pfns returned by follow_fault_pfn to struct pages to transiently take a reference. The reference was originally taken to match the reference taken by gup. However, pfns returned by follow_fault_pfn may not have a struct page set up for reference counting.
Signed-off-by: David Stevens stevensd@chromium.org --- arch/arm64/kvm/mmu.c | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 896b3644b36f..a741972cb75f 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -779,17 +779,17 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, */ static unsigned long transparent_hugepage_adjust(struct kvm_memory_slot *memslot, - unsigned long hva, kvm_pfn_t *pfnp, + unsigned long hva, struct kvm_pfn_page *pfnpgp, phys_addr_t *ipap) { - kvm_pfn_t pfn = *pfnp; + kvm_pfn_t pfn = pfnpgp->pfn;
/* * Make sure the adjustment is done only for THP pages. Also make * sure that the HVA and IPA are sufficiently aligned and that the * block map is contained within the memslot. */ - if (kvm_is_transparent_hugepage(pfn) && + if (pfnpgp->page && kvm_is_transparent_hugepage(pfn) && fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { /* * The address we faulted on is backed by a transparent huge @@ -810,10 +810,11 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, * page accordingly. */ *ipap &= PMD_MASK; - kvm_release_pfn_clean(pfn); + put_page(pfnpgp->page); pfn &= ~(PTRS_PER_PMD - 1); - kvm_get_pfn(pfn); - *pfnp = pfn; + pfnpgp->pfn = pfn; + pfnpgp->page = pfn_to_page(pfnpgp->pfn); + get_page(pfnpgp->page);
return PMD_SIZE; } @@ -836,7 +837,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct vm_area_struct *vma; short vma_shift; gfn_t gfn; - kvm_pfn_t pfn; + struct kvm_pfn_page pfnpg; bool logging_active = memslot_is_logging(memslot); unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu); unsigned long vma_pagesize, fault_granule; @@ -933,17 +934,16 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, */ smp_rmb();
- pfn = kvm_pfn_page_unwrap(__gfn_to_pfn_memslot(memslot, gfn, false, - NULL, write_fault, - &writable, NULL)); - if (pfn == KVM_PFN_ERR_HWPOISON) { + pfnpg = __gfn_to_pfn_memslot(memslot, gfn, false, NULL, + write_fault, &writable, NULL); + if (pfnpg.pfn == KVM_PFN_ERR_HWPOISON) { kvm_send_hwpoison_signal(hva, vma_shift); return 0; } - if (is_error_noslot_pfn(pfn)) + if (is_error_noslot_pfn(pfnpg.pfn)) return -EFAULT;
- if (kvm_is_device_pfn(pfn)) { + if (kvm_is_device_pfn(pfnpg.pfn)) { device = true; force_pte = true; } else if (logging_active && !write_fault) { @@ -968,16 +968,16 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, */ if (vma_pagesize == PAGE_SIZE && !force_pte) vma_pagesize = transparent_hugepage_adjust(memslot, hva, - &pfn, &fault_ipa); + &pfnpg, &fault_ipa); if (writable) prot |= KVM_PGTABLE_PROT_W;
if (fault_status != FSC_PERM && !device) - clean_dcache_guest_page(pfn, vma_pagesize); + clean_dcache_guest_page(pfnpg.pfn, vma_pagesize);
if (exec_fault) { prot |= KVM_PGTABLE_PROT_X; - invalidate_icache_guest_page(pfn, vma_pagesize); + invalidate_icache_guest_page(pfnpg.pfn, vma_pagesize); }
if (device) @@ -994,20 +994,23 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot); } else { ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize, - __pfn_to_phys(pfn), prot, + __pfn_to_phys(pfnpg.pfn), prot, memcache); }
/* Mark the page dirty only if the fault is handled successfully */ if (writable && !ret) { - kvm_set_pfn_dirty(pfn); + if (pfnpg.page) + kvm_set_pfn_dirty(pfnpg.pfn); mark_page_dirty_in_slot(kvm, memslot, gfn); }
out_unlock: spin_unlock(&kvm->mmu_lock); - kvm_set_pfn_accessed(pfn); - kvm_release_pfn_clean(pfn); + if (pfnpg.page) { + kvm_set_pfn_accessed(pfnpg.pfn); + put_page(pfnpg.page); + } return ret != -EAGAIN ? ret : 0; }