Adds a basic test of the HMM unmap and pin operation.
Signed-off-by: Alistair Popple apopple@nvidia.com --- lib/test_hmm.c | 107 +++++++++++++++++++++---- lib/test_hmm_uapi.h | 1 + tools/testing/selftests/vm/hmm-tests.c | 49 +++++++++++ 3 files changed, 140 insertions(+), 17 deletions(-)
diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 98848b96ff09..c78a473250a3 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -46,6 +46,7 @@ struct dmirror_bounce { unsigned long cpages; };
+#define DPT_XA_TAG_ATOMIC 1UL #define DPT_XA_TAG_WRITE 3UL
/* @@ -83,6 +84,7 @@ struct dmirror_device { struct cdev cdevice; struct hmm_devmem *devmem;
+ unsigned int devmem_faults; unsigned int devmem_capacity; unsigned int devmem_count; struct dmirror_chunk **devmem_chunks; @@ -203,8 +205,18 @@ static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, * Therefore, it is OK to just clear the entry. */ xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, - end >> PAGE_SHIFT) + end >> PAGE_SHIFT) { + /* + * Typically this would be done in devmap free page, but as + * we're using the XArray to store the reference to the original + * page do it here as it doesn't matter if clean up of the + * pinned page is delayed. + */ + if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) + unpin_user_page(xa_untag_pointer(entry)); + xa_erase(&dmirror->pt, pfn); + } }
static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, @@ -571,7 +583,8 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) }
static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, - struct dmirror *dmirror) + struct dmirror *dmirror, + int allow_ref) { struct dmirror_device *mdevice = dmirror->mdevice; const unsigned long *src = args->src; @@ -598,9 +611,17 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, continue;
rpage = dpage->zone_device_data; - if (spage) + if (spage && !(*src & MIGRATE_PFN_PIN)) copy_highpage(rpage, spage); else + /* + * In the MIGRATE_PFN_PIN case we don't really + * need rpage at all because the existing page is + * staying in place and will be mapped. However we need + * somewhere to store dmirror and that place is + * rpage->zone_device_data so we keep it for + * simplicity. + */ clear_highpage(rpage);
/* @@ -620,7 +641,8 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, }
static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, - struct dmirror *dmirror) + struct dmirror *dmirror, + int allow_ref) { unsigned long start = args->start; unsigned long end = args->end; @@ -647,8 +669,14 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, * Store the page that holds the data so the page table * doesn't have to deal with ZONE_DEVICE private pages. */ - entry = dpage->zone_device_data; - if (*dst & MIGRATE_PFN_WRITE) + if (*src & MIGRATE_PFN_PIN) + entry = migrate_pfn_to_page(*src); + else + entry = dpage->zone_device_data; + + if (*src & MIGRATE_PFN_PIN) + entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC); + else if (*dst & MIGRATE_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); if (xa_is_err(entry)) { @@ -662,7 +690,8 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, }
static int dmirror_migrate(struct dmirror *dmirror, - struct hmm_dmirror_cmd *cmd) + struct hmm_dmirror_cmd *cmd, + int allow_ref) { unsigned long start, end, addr; unsigned long size = cmd->npages << PAGE_SHIFT; @@ -673,7 +702,7 @@ static int dmirror_migrate(struct dmirror *dmirror, struct dmirror_bounce bounce; struct migrate_vma args; unsigned long next; - int ret; + int i, ret;
start = cmd->addr; end = start + size; @@ -696,8 +725,13 @@ static int dmirror_migrate(struct dmirror *dmirror, if (next > vma->vm_end) next = vma->vm_end;
- memset(src_pfns, 0, ARRAY_SIZE(src_pfns)); - memset(dst_pfns, 0, ARRAY_SIZE(dst_pfns)); + if (allow_ref) + for (i = 0; i < 64; ++i) + src_pfns[i] = MIGRATE_PFN_PIN; + else + memset(src_pfns, 0, sizeof(src_pfns)); + memset(dst_pfns, 0, sizeof(dst_pfns)); + args.vma = vma; args.src = src_pfns; args.dst = dst_pfns; @@ -709,9 +743,9 @@ static int dmirror_migrate(struct dmirror *dmirror, if (ret) goto out;
- dmirror_migrate_alloc_and_copy(&args, dmirror); + dmirror_migrate_alloc_and_copy(&args, dmirror, allow_ref); migrate_vma_pages(&args); - dmirror_migrate_finalize_and_map(&args, dmirror); + dmirror_migrate_finalize_and_map(&args, dmirror, allow_ref); migrate_vma_finalize(&args); } mmap_read_unlock(mm); @@ -739,6 +773,28 @@ static int dmirror_migrate(struct dmirror *dmirror, return ret; }
+static int dmirror_migrate_pin(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd) +{ + void *tmp; + int nr_pages = cmd->npages; + int ret; + + ret = dmirror_migrate(dmirror, cmd, true); + + tmp = kmalloc(nr_pages << PAGE_SHIFT, GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + /* Make sure user access faults */ + dmirror->mdevice->devmem_faults = 0; + if (copy_from_user(tmp, u64_to_user_ptr(cmd->addr), nr_pages << PAGE_SHIFT)) + ret = -EFAULT; + cmd->faults = dmirror->mdevice->devmem_faults; + + return ret; +} + static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, unsigned char *perm, unsigned long entry) { @@ -948,7 +1004,11 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, break;
case HMM_DMIRROR_MIGRATE: - ret = dmirror_migrate(dmirror, &cmd); + ret = dmirror_migrate(dmirror, &cmd, false); + break; + + case HMM_DMIRROR_MIGRATE_PIN: + ret = dmirror_migrate_pin(dmirror, &cmd); break;
case HMM_DMIRROR_SNAPSHOT: @@ -1004,20 +1064,31 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, for (addr = start; addr < end; addr += PAGE_SIZE, src++, dst++) { struct page *dpage, *spage; + void *entry;
spage = migrate_pfn_to_page(*src); if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) continue; - spage = spage->zone_device_data;
- dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); + entry = xa_load(&dmirror->pt, addr >> PAGE_SHIFT); + if (entry && xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) { + spage = NULL; + dpage = xa_untag_pointer(entry); + *dst = migrate_pfn(page_to_pfn(dpage)) | + MIGRATE_PFN_LOCKED | MIGRATE_PFN_UNPIN; + } else { + spage = spage->zone_device_data; + dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); + *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; + } + if (!dpage) continue;
lock_page(dpage); xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); - copy_highpage(dpage, spage); - *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; + if (spage) + copy_highpage(dpage, spage); if (*src & MIGRATE_PFN_WRITE) *dst |= MIGRATE_PFN_WRITE; } @@ -1041,6 +1112,8 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) rpage = vmf->page->zone_device_data; dmirror = rpage->zone_device_data;
+ dmirror->mdevice->devmem_faults++; + /* FIXME demonstrate how we can adjust migrate range */ args.vma = vmf->vma; args.start = vmf->address; diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index 670b4ef2a5b6..b40f4e6affe0 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -33,6 +33,7 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_WRITE _IOWR('H', 0x01, struct hmm_dmirror_cmd) #define HMM_DMIRROR_MIGRATE _IOWR('H', 0x02, struct hmm_dmirror_cmd) #define HMM_DMIRROR_SNAPSHOT _IOWR('H', 0x03, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_MIGRATE_PIN _IOWR('H', 0x04, struct hmm_dmirror_cmd)
/* * Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT. diff --git a/tools/testing/selftests/vm/hmm-tests.c b/tools/testing/selftests/vm/hmm-tests.c index 5d1ac691b9f4..7111ebab93c7 100644 --- a/tools/testing/selftests/vm/hmm-tests.c +++ b/tools/testing/selftests/vm/hmm-tests.c @@ -947,6 +947,55 @@ TEST_F(hmm, migrate_fault) hmm_buffer_free(buffer); }
+TEST_F(hmm, migrate_fault_pin) +{ + struct hmm_buffer *buffer; + unsigned long npages; + unsigned long size; + unsigned long i; + int *ptr; + int ret; + + npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; + ASSERT_NE(npages, 0); + size = npages << self->page_shift; + + buffer = malloc(sizeof(*buffer)); + ASSERT_NE(buffer, NULL); + + buffer->fd = -1; + buffer->size = size; + buffer->mirror = malloc(size); + ASSERT_NE(buffer->mirror, NULL); + + buffer->ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + buffer->fd, 0); + ASSERT_NE(buffer->ptr, MAP_FAILED); + + /* Initialize buffer in system memory. */ + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ptr[i] = i; + + /* Migrate memory to device. */ + ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE_PIN, buffer, npages); + ASSERT_EQ(ret, 0); + ASSERT_EQ(buffer->cpages, npages); + + /* Check what the device read. */ + for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i], i); + + ASSERT_EQ(buffer->faults, npages); + + /* Fault pages back to system memory and check them. */ + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i], i); + + hmm_buffer_free(buffer); +} + /* * Migrate anonymous shared memory to device private memory. */