Sometimes you might want to use MAP_POPULATE to ask a device driver to initialize the device memory in some specific manner. SGX driver can use this to request more memory by issuing ENCLS[EAUG] x86 opcode for each page in the address range.
Add f_ops->populate() with the same parameters as f_ops->mmap() and make it conditionally called inside call_mmap(). Update call sites accodingly.
Signed-off-by: Jarkko Sakkinen jarkko@kernel.org --- v2: - if (!ret && do_populate) + if (!ret && do_populate && file->f_op->populate) (reported by Jan Harkes) --- arch/mips/kernel/vdso.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c | 2 +- fs/coda/file.c | 2 +- fs/overlayfs/file.c | 2 +- include/linux/fs.h | 10 ++++++++-- include/linux/mm.h | 2 +- ipc/shm.c | 2 +- mm/mmap.c | 10 +++++----- mm/nommu.c | 4 ++-- 9 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index 3d0cf471f2fe..89f3f3da9abd 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -102,7 +102,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) base = mmap_region(NULL, STACK_TOP, PAGE_SIZE, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, - 0, NULL); + 0, NULL, false); if (IS_ERR_VALUE(base)) { ret = base; goto out; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index 1b526039a60d..4c71f64d6a79 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -107,7 +107,7 @@ static int i915_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct * if (!obj->base.filp) return -ENODEV;
- ret = call_mmap(obj->base.filp, vma); + ret = call_mmap(obj->base.filp, vma, false); if (ret) return ret;
diff --git a/fs/coda/file.c b/fs/coda/file.c index 29dd87be2fb8..e14f312fdbf8 100644 --- a/fs/coda/file.c +++ b/fs/coda/file.c @@ -173,7 +173,7 @@ coda_file_mmap(struct file *coda_file, struct vm_area_struct *vma) spin_unlock(&cii->c_lock);
vma->vm_file = get_file(host_file); - ret = call_mmap(vma->vm_file, vma); + ret = call_mmap(vma->vm_file, vma, false);
if (ret) { /* if call_mmap fails, our caller will put host_file so we diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index fa125feed0ff..b963a9397e80 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -503,7 +503,7 @@ static int ovl_mmap(struct file *file, struct vm_area_struct *vma) vma_set_file(vma, realfile);
old_cred = ovl_override_creds(file_inode(file)->i_sb); - ret = call_mmap(vma->vm_file, vma); + ret = call_mmap(vma->vm_file, vma, false); revert_creds(old_cred); ovl_file_accessed(file);
diff --git a/include/linux/fs.h b/include/linux/fs.h index e2d892b201b0..4c6a3339373d 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1993,6 +1993,7 @@ struct file_operations { long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*compat_ioctl) (struct file *, unsigned int, unsigned long); int (*mmap) (struct file *, struct vm_area_struct *); + int (*populate)(struct file *, struct vm_area_struct *); unsigned long mmap_supported_flags; int (*open) (struct inode *, struct file *); int (*flush) (struct file *, fl_owner_t id); @@ -2074,9 +2075,14 @@ static inline ssize_t call_write_iter(struct file *file, struct kiocb *kio, return file->f_op->write_iter(kio, iter); }
-static inline int call_mmap(struct file *file, struct vm_area_struct *vma) +static inline int call_mmap(struct file *file, struct vm_area_struct *vma, bool do_populate) { - return file->f_op->mmap(file, vma); + int ret = file->f_op->mmap(file, vma); + + if (!ret && do_populate && file->f_op->populate) + ret = file->f_op->populate(file, vma); + + return ret; }
extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); diff --git a/include/linux/mm.h b/include/linux/mm.h index 213cc569b192..6c8c036f423b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2683,7 +2683,7 @@ extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned lo
extern unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, - struct list_head *uf); + struct list_head *uf, bool do_populate); extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); diff --git a/ipc/shm.c b/ipc/shm.c index b3048ebd5c31..89b28f32acf0 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -587,7 +587,7 @@ static int shm_mmap(struct file *file, struct vm_area_struct *vma) if (ret) return ret;
- ret = call_mmap(sfd->file, vma); + ret = call_mmap(sfd->file, vma, do_populate); if (ret) { shm_close(vma); return ret; diff --git a/mm/mmap.c b/mm/mmap.c index 1e8fdb0b51ed..5eca79957d4c 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1413,6 +1413,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf) { + bool do_populate = (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE; struct mm_struct *mm = current->mm; vm_flags_t vm_flags; int pkey = 0; @@ -1579,10 +1580,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_NORESERVE; }
- addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vm_flags, pgoff, uf, do_populate); if (!IS_ERR_VALUE(addr) && - ((vm_flags & VM_LOCKED) || - (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) + ((vm_flags & VM_LOCKED) || do_populate)) *populate = len; return addr; } @@ -1721,7 +1721,7 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, - struct list_head *uf) + struct list_head *uf, bool do_populate) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev, *merge; @@ -1790,7 +1790,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, }
vma->vm_file = get_file(file); - error = call_mmap(file, vma); + error = call_mmap(file, vma, do_populate); if (error) goto unmap_and_free_vma;
diff --git a/mm/nommu.c b/mm/nommu.c index 55a9e48a7a02..a3c20b803c27 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -941,7 +941,7 @@ static int do_mmap_shared_file(struct vm_area_struct *vma) { int ret;
- ret = call_mmap(vma->vm_file, vma); + ret = call_mmap(vma->vm_file, vma, false); if (ret == 0) { vma->vm_region->vm_top = vma->vm_region->vm_end; return 0; @@ -972,7 +972,7 @@ static int do_mmap_private(struct vm_area_struct *vma, * - VM_MAYSHARE will be set if it may attempt to share */ if (capabilities & NOMMU_MAP_DIRECT) { - ret = call_mmap(vma->vm_file, vma); + ret = call_mmap(vma->vm_file, vma, false); if (ret == 0) { /* shouldn't return success if we're not sharing */ BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
On Sun, 6 Mar 2022 05:26:55 +0200 Jarkko Sakkinen jarkko@kernel.org wrote:
Sometimes you might want to use MAP_POPULATE to ask a device driver to initialize the device memory in some specific manner. SGX driver can use this to request more memory by issuing ENCLS[EAUG] x86 opcode for each page in the address range.
Why is this useful? Please fully describe the benefit to kernel users. Convince us that the benefit justifies the code churn, maintenance cost and larger kernel footprint.
Do we know of any other drivers which might use this?
Add f_ops->populate() with the same parameters as f_ops->mmap() and make it conditionally called inside call_mmap(). Update call sites accodingly.
spello
-static inline int call_mmap(struct file *file, struct vm_area_struct *vma) +static inline int call_mmap(struct file *file, struct vm_area_struct *vma, bool do_populate) {
- return file->f_op->mmap(file, vma);
- int ret = file->f_op->mmap(file, vma);
- if (!ret && do_populate && file->f_op->populate)
ret = file->f_op->populate(file, vma);- return ret;
}
Should this still be inlined?
On 3/6/22 15:24, Andrew Morton wrote:
On Sun, 6 Mar 2022 05:26:55 +0200 Jarkko Sakkinen jarkko@kernel.org wrote:
Sometimes you might want to use MAP_POPULATE to ask a device driver to initialize the device memory in some specific manner. SGX driver can use this to request more memory by issuing ENCLS[EAUG] x86 opcode for each page in the address range.
Why is this useful? Please fully describe the benefit to kernel users. Convince us that the benefit justifies the code churn, maintenance cost and larger kernel footprint.
In short: page faults stink. The core kernel has lots of ways of avoiding page faults like madvise(MADV_WILLNEED) or mmap(MAP_POPULATE). But, those only work on normal RAM that the core mm manages.
SGX is weird. SGX memory is managed outside the core mm. It doesn't have a 'struct page' and get_user_pages() doesn't work on it. Its VMAs are marked with VM_IO. So, none of the existing methods for avoiding page faults work on SGX memory.
This essentially helps extend existing "normal RAM" kernel ABIs to work for avoiding faults for SGX too. SGX users want to enjoy all of the benefits of a delayed allocation policy (better resource use, overcommit, NUMA affinity) but without the cost of millions of faults.
That said, this isn't how I would have implemented it. I probably would have hooked in to populate_vma_page_range() or its callers.
On Sun, Mar 06, 2022 at 03:41:54PM -0800, Dave Hansen wrote:
On 3/6/22 15:24, Andrew Morton wrote:
On Sun, 6 Mar 2022 05:26:55 +0200 Jarkko Sakkinen jarkko@kernel.org wrote:
Sometimes you might want to use MAP_POPULATE to ask a device driver to initialize the device memory in some specific manner. SGX driver can use this to request more memory by issuing ENCLS[EAUG] x86 opcode for each page in the address range.
Why is this useful? Please fully describe the benefit to kernel users. Convince us that the benefit justifies the code churn, maintenance cost and larger kernel footprint.
In short: page faults stink. The core kernel has lots of ways of avoiding page faults like madvise(MADV_WILLNEED) or mmap(MAP_POPULATE). But, those only work on normal RAM that the core mm manages.
SGX is weird. SGX memory is managed outside the core mm. It doesn't have a 'struct page' and get_user_pages() doesn't work on it. Its VMAs are marked with VM_IO. So, none of the existing methods for avoiding page faults work on SGX memory.
This essentially helps extend existing "normal RAM" kernel ABIs to work for avoiding faults for SGX too. SGX users want to enjoy all of the benefits of a delayed allocation policy (better resource use, overcommit, NUMA affinity) but without the cost of millions of faults.
That said, this isn't how I would have implemented it. I probably would have hooked in to populate_vma_page_range() or its callers.
The exact implementation path is not driver in this. I'm open for better options. The point of these patches is more to show an issue rather than solution, and they do carry RFC because of that.
Hooking into populate_vma_page_range() does sound like a better idea, because then it would be nicely embedded into __mm_populate() and other functionality that calls that function.
But e.g. in __mm_populate() anything with (VM_IO | VM_PFNMAP) gets filtered out and never reach that function.
I don't know unorthodox that'd be but could we perhaps have a VM flag for SGX?
BR, Jarkko
On 3/7/22 03:27, Jarkko Sakkinen wrote:
But e.g. in __mm_populate() anything with (VM_IO | VM_PFNMAP) gets filtered out and never reach that function.
I don't know unorthodox that'd be but could we perhaps have a VM flag for SGX?
SGX only works on a subset of the chips from one vendor on one architecture. That doesn't seem worth burning a VM flag.
On Mon, Mar 07, 2022 at 07:29:22AM -0800, Dave Hansen wrote:
On 3/7/22 03:27, Jarkko Sakkinen wrote:
But e.g. in __mm_populate() anything with (VM_IO | VM_PFNMAP) gets filtered out and never reach that function.
I don't know unorthodox that'd be but could we perhaps have a VM flag for SGX?
SGX only works on a subset of the chips from one vendor on one architecture. That doesn't seem worth burning a VM flag.
What do you think of Matthew's idea of using ra_state for prediction?
BR, Jarkko
On Sun, Mar 06, 2022 at 03:41:54PM -0800, Dave Hansen wrote:
In short: page faults stink. The core kernel has lots of ways of avoiding page faults like madvise(MADV_WILLNEED) or mmap(MAP_POPULATE). But, those only work on normal RAM that the core mm manages.
SGX is weird. SGX memory is managed outside the core mm. It doesn't have a 'struct page' and get_user_pages() doesn't work on it. Its VMAs are marked with VM_IO. So, none of the existing methods for avoiding page faults work on SGX memory.
This essentially helps extend existing "normal RAM" kernel ABIs to work for avoiding faults for SGX too. SGX users want to enjoy all of the benefits of a delayed allocation policy (better resource use, overcommit, NUMA affinity) but without the cost of millions of faults.
We have a mechanism for dynamically reducing the number of page faults already; it's just buried in the page cache code. You have vma->vm_file, which contains a file_ra_state. You can use this to track where recent faults have been and grow the size of the region you fault in per page fault. You don't have to (indeed probably don't want to) use the same algorithm as the page cache, but the _principle_ is the same -- were recent speculative faults actually used; should we grow the number of pages actually faulted in, or is this a random sparse workload where we want to allocate individual pages.
Don't rely on the user to ask. They don't know.
On Mon, Mar 07, 2022 at 02:37:48PM +0000, Matthew Wilcox wrote:
On Sun, Mar 06, 2022 at 03:41:54PM -0800, Dave Hansen wrote:
In short: page faults stink. The core kernel has lots of ways of avoiding page faults like madvise(MADV_WILLNEED) or mmap(MAP_POPULATE). But, those only work on normal RAM that the core mm manages.
SGX is weird. SGX memory is managed outside the core mm. It doesn't have a 'struct page' and get_user_pages() doesn't work on it. Its VMAs are marked with VM_IO. So, none of the existing methods for avoiding page faults work on SGX memory.
This essentially helps extend existing "normal RAM" kernel ABIs to work for avoiding faults for SGX too. SGX users want to enjoy all of the benefits of a delayed allocation policy (better resource use, overcommit, NUMA affinity) but without the cost of millions of faults.
We have a mechanism for dynamically reducing the number of page faults already; it's just buried in the page cache code. You have vma->vm_file, which contains a file_ra_state. You can use this to track where recent faults have been and grow the size of the region you fault in per page fault. You don't have to (indeed probably don't want to) use the same algorithm as the page cache, but the _principle_ is the same -- were recent speculative faults actually used; should we grow the number of pages actually faulted in, or is this a random sparse workload where we want to allocate individual pages.
Don't rely on the user to ask. They don't know.
This sounds like a possibility. I'll need to study it properly first though. Thank you for pointing this out.
BR, Jarkko
On Sun, Mar 06, 2022 at 03:24:56PM -0800, Andrew Morton wrote:
On Sun, 6 Mar 2022 05:26:55 +0200 Jarkko Sakkinen jarkko@kernel.org wrote:
Sometimes you might want to use MAP_POPULATE to ask a device driver to initialize the device memory in some specific manner. SGX driver can use this to request more memory by issuing ENCLS[EAUG] x86 opcode for each page in the address range.
Why is this useful? Please fully describe the benefit to kernel users. Convince us that the benefit justifies the code churn, maintenance cost and larger kernel footprint.
Do we know of any other drivers which might use this?
Brutal honesty: I don't know if any other drivers would use this but neither I would not be surprised if they did. The need for this might very well be "masked" by ioctl API's. I was first proposing a ioctl for this but Dave suggested to at least try out this route.
Add f_ops->populate() with the same parameters as f_ops->mmap() and make it conditionally called inside call_mmap(). Update call sites accodingly.
spello
Thanks, I noticed that but did not want to spam with a new version just because of that :-)
-static inline int call_mmap(struct file *file, struct vm_area_struct *vma) +static inline int call_mmap(struct file *file, struct vm_area_struct *vma, bool do_populate) {
- return file->f_op->mmap(file, vma);
- int ret = file->f_op->mmap(file, vma);
- if (!ret && do_populate && file->f_op->populate)
ret = file->f_op->populate(file, vma);- return ret;
}
Should this still be inlined?
I think it might make sense at least to have call_mmap_populate() so and mmap_region_populate() instead of putting that boolean parameter to every flow (based on Greg's feedback). But only if this implementation approach is used in the first place.
As said, I chose to use RFC to pinpoint a bottleneck for us, not claiming that this would be the best possible way to work around it.
BR, Jarkko
dri-devel@lists.freedesktop.org
-
Andrew Morton -
Dave Hansen -
Jarkko Sakkinen -
Matthew Wilcox