On Wed, Oct 30, 2019 at 03:49:12PM -0700, John Hubbard wrote:

A subsequent patch requires access to gup flags, so pass the flags argument through to the __gup_device_* functions.

Also placate checkpatch.pl by shortening a nearby line.

Reviewed-by: Ira Weiny ira.weiny@intel.com

Cc: Kirill A. Shutemov kirill.shutemov@linux.intel.com Signed-off-by: John Hubbard jhubbard@nvidia.com

---

mm/gup.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/mm/gup.c b/mm/gup.c index 8f236a335ae9..85caf76b3012 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1890,7 +1890,8 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,

#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE) static int __gup_device_huge(unsigned long pfn, unsigned long addr,

• unsigned long end, struct page **pages, int *nr)
  

• 	     unsigned long end, unsigned int flags,
  

• 	     struct page **pages, int *nr)
  

{ int nr_start = *nr; struct dev_pagemap *pgmap = NULL; @@ -1916,13 +1917,14 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr, }

static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,

• unsigned long end, struct page **pages, int *nr)
  

• 		 unsigned long end, unsigned int flags,
  

• 		 struct page **pages, int *nr)
  

{ unsigned long fault_pfn; int nr_start = *nr;

fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);

• if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))

• if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) return 0;

  if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {

@@ -1933,13 +1935,14 @@ static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, }

static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,

• unsigned long end, struct page **pages, int *nr)
  

• 		 unsigned long end, unsigned int flags,
  

• 		 struct page **pages, int *nr)
  

{ unsigned long fault_pfn; int nr_start = *nr;

fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);

• if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))

• if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) return 0;

  if (unlikely(pud_val(orig) != pud_val(*pudp))) {

@@ -1950,14 +1953,16 @@ static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, } #else static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,

• unsigned long end, struct page **pages, int *nr)
  

• 		 unsigned long end, unsigned int flags,
  

• 		 struct page **pages, int *nr)
  

{ BUILD_BUG(); return 0; }

static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,

• unsigned long end, struct page **pages, int *nr)
  

• 		 unsigned long end, unsigned int flags,
  

• 		 struct page **pages, int *nr)
  

{ BUILD_BUG(); return 0; @@ -2062,7 +2067,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, if (pmd_devmap(orig)) { if (unlikely(flags & FOLL_LONGTERM)) return 0;

• return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr);
  

• return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
  

• 			     pages, nr);
  

  }

  refs = 0;

@@ -2092,7 +2098,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, }

static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,

• unsigned long end, unsigned int flags, struct page **pages, int *nr)
  

• 	unsigned long end, unsigned int flags,
  

• 	struct page **pages, int *nr)
  

{ struct page *head, *page; int refs; @@ -2103,7 +2110,8 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, if (pud_devmap(orig)) { if (unlikely(flags & FOLL_LONGTERM)) return 0;

• return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr);
  

• return __gup_device_huge_pud(orig, pudp, addr, end, flags,
  

• 			     pages, nr);
  

  }

  refs = 0;

-- 2.23.0

On Wed, Oct 30, 2019 at 03:49:13PM -0700, John Hubbard wrote:

There are four locations in gup.c that have a fair amount of code duplication. This means that changing one requires making the same changes in four places, not to mention reading the same code four times, and wondering if there are subtle differences.

Factor out the common code into static functions, thus reducing the overall line count and the code's complexity.

Also, take the opportunity to slightly improve the efficiency of the error cases, by doing a mass subtraction of the refcount, surrounded by get_page()/put_page().

Also, further simplify (slightly), by waiting until the the successful end of each routine, to increment *nr.

Overall it seems like a pretty good clean up. It did take a bit of review but I _think_ it is correct. A couple of comments below.

Cc: Christoph Hellwig hch@lst.de Cc: Aneesh Kumar K.V aneesh.kumar@linux.ibm.com Signed-off-by: John Hubbard jhubbard@nvidia.com

---

mm/gup.c | 113 ++++++++++++++++++++++--------------------------------- 1 file changed, 46 insertions(+), 67 deletions(-)

diff --git a/mm/gup.c b/mm/gup.c index 85caf76b3012..8fb0d9cdfaf5 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1969,6 +1969,35 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, } #endif

+static int __record_subpages(struct page *page, unsigned long addr,

• 	     unsigned long end, struct page **pages, int nr)
  

+{

• int nr_recorded_pages = 0;
• do {

• pages[nr] = page;
  

• nr++;
  

• page++;
  

• nr_recorded_pages++;
  

• } while (addr += PAGE_SIZE, addr != end);
• return nr_recorded_pages;

+}

+static void __remove_refs_from_head(struct page *page, int refs) +{

• /* Do a get_page() first, in case refs == page->_refcount */
• get_page(page);
• page_ref_sub(page, refs);
• put_page(page);

+}

I wonder if this is better implemented as "put_compound_head()"? To match the try_get_compound_head() call below?

+static int __huge_pt_done(struct page *head, int nr_recorded_pages, int *nr) +{

• *nr += nr_recorded_pages;
• SetPageReferenced(head);
• return 1;

When will this return anything but 1?

Ira

+}

#ifdef CONFIG_ARCH_HAS_HUGEPD static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, unsigned long sz) @@ -1998,34 +2027,19 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, /* hugepages are never "special" */ VM_BUG_ON(!pfn_valid(pte_pfn(pte)));

• refs = 0; head = pte_page(pte);
• page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
• do {

• VM_BUG_ON(compound_head(page) != head);
  

• pages[*nr] = page;
  

• (*nr)++;
  

• page++;
  

• refs++;
  

• } while (addr += PAGE_SIZE, addr != end);

• refs = __record_subpages(page, addr, end, pages, *nr);

  head = try_get_compound_head(head, refs);

• if (!head) {

• *nr -= refs;
  

• if (!head) return 0;

• }

  if (unlikely(pte_val(pte) != pte_val(*ptep))) {

• /* Could be optimized better */
  

• *nr -= refs;
  

• while (refs--)
  

• 	put_page(head);
  

• __remove_refs_from_head(head, refs);
  

  return 0; }

• SetPageReferenced(head);
• return 1;

• return __huge_pt_done(head, refs, nr);

}

static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, @@ -2071,30 +2085,18 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, pages, nr); }

• refs = 0; page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
• do {

• pages[*nr] = page;
  

• (*nr)++;
  

• page++;
  

• refs++;
  

• } while (addr += PAGE_SIZE, addr != end);

• refs = __record_subpages(page, addr, end, pages, *nr);

  head = try_get_compound_head(pmd_page(orig), refs);

• if (!head) {

• *nr -= refs;
  

• if (!head) return 0;

• }

  if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {

• *nr -= refs;
  

• while (refs--)
  

• 	put_page(head);
  

• __remove_refs_from_head(head, refs);
  

  return 0; }

• SetPageReferenced(head);
• return 1;

• return __huge_pt_done(head, refs, nr);

}

static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, @@ -2114,30 +2116,18 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, pages, nr); }

• refs = 0; page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
• do {

• pages[*nr] = page;
  

• (*nr)++;
  

• page++;
  

• refs++;
  

• } while (addr += PAGE_SIZE, addr != end);

• refs = __record_subpages(page, addr, end, pages, *nr);

  head = try_get_compound_head(pud_page(orig), refs);

• if (!head) {

• *nr -= refs;
  

• if (!head) return 0;

• }

  if (unlikely(pud_val(orig) != pud_val(*pudp))) {

• *nr -= refs;
  

• while (refs--)
  

• 	put_page(head);
  

• __remove_refs_from_head(head, refs);
  

  return 0; }

• SetPageReferenced(head);
• return 1;

• return __huge_pt_done(head, refs, nr);

}

static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, @@ -2151,30 +2141,19 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, return 0;

BUILD_BUG_ON(pgd_devmap(orig));

• refs = 0;

• page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);

• do {

• pages[*nr] = page;
  

• (*nr)++;
  

• page++;
  

• refs++;
  

• } while (addr += PAGE_SIZE, addr != end);

• refs = __record_subpages(page, addr, end, pages, *nr);

  head = try_get_compound_head(pgd_page(orig), refs);

• if (!head) {

• *nr -= refs;
  

• if (!head) return 0;

• }

  if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {

• *nr -= refs;
  

• while (refs--)
  

• 	put_page(head);
  

• __remove_refs_from_head(head, refs);
  

  return 0; }

• SetPageReferenced(head);
• return 1;

• return __huge_pt_done(head, refs, nr);

}

static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,

2.23.0

On Wed, Oct 30, 2019 at 03:49:16PM -0700, John Hubbard wrote:

Introduce pin_user_pages*() variations of get_user_pages*() calls, and also pin_longterm_pages*() variations.

These variants all set FOLL_PIN, which is also introduced, and basically documented. (An upcoming patch provides more extensive documentation.) The second set (pin_longterm*) also sets FOLL_LONGTERM:

pin_user_pages()
pin_user_pages_remote()
pin_user_pages_fast()

pin_longterm_pages()
pin_longterm_pages_remote()
pin_longterm_pages_fast()

All pages that are pinned via the above calls, must be unpinned via put_user_page().

The underlying rules are:

• These are gup-internal flags, so the call sites should not directly

set FOLL_PIN nor FOLL_LONGTERM. That behavior is enforced with assertions, for the new FOLL_PIN flag. However, for the pre-existing FOLL_LONGTERM flag, which has some call sites that still directly set FOLL_LONGTERM, there is no assertion yet.

• Call sites that want to indicate that they are going to do DirectIO ("DIO") or something with similar characteristics, should call a get_user_pages()-like wrapper call that sets FOLL_PIN. These wrappers will: * Start with "pin_user_pages" instead of "get_user_pages". That makes it easy to find and audit the call sites. * Set FOLL_PIN

• For pages that are received via FOLL_PIN, those pages must be returned via put_user_page().

Signed-off-by: John Hubbard jhubbard@nvidia.com

include/linux/mm.h | 53 ++++++++- mm/gup.c | 284 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 311 insertions(+), 26 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h index cc292273e6ba..62c838a3e6c7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1526,9 +1526,23 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas, int *locked); +long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,

• 	   unsigned long start, unsigned long nr_pages,
  

• 	   unsigned int gup_flags, struct page **pages,
  

• 	   struct vm_area_struct **vmas, int *locked);
  

+long pin_longterm_pages_remote(struct task_struct *tsk, struct mm_struct *mm,

• 	       unsigned long start, unsigned long nr_pages,
  

• 	       unsigned int gup_flags, struct page **pages,
  

• 	       struct vm_area_struct **vmas, int *locked);
  

long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas); +long pin_user_pages(unsigned long start, unsigned long nr_pages,

•     unsigned int gup_flags, struct page **pages,
  

•     struct vm_area_struct **vmas);
  

+long pin_longterm_pages(unsigned long start, unsigned long nr_pages,

• 	unsigned int gup_flags, struct page **pages,
  

• 	struct vm_area_struct **vmas);
  

long get_user_pages_locked(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked); long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, @@ -1536,6 +1550,10 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,

int get_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages); +int pin_user_pages_fast(unsigned long start, int nr_pages,

• 	unsigned int gup_flags, struct page **pages);
  

+int pin_longterm_pages_fast(unsigned long start, int nr_pages,

• 	    unsigned int gup_flags, struct page **pages);
  
  

int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, @@ -2594,13 +2612,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, #define FOLL_ANON 0x8000 /* don't do file mappings */ #define FOLL_LONGTERM 0x10000 /* mapping lifetime is indefinite: see below */ #define FOLL_SPLIT_PMD 0x20000 /* split huge pmd before returning */ +#define FOLL_PIN 0x40000 /* pages must be released via put_user_page() */

/*

• • NOTE on FOLL_LONGTERM:

• • FOLL_PIN and FOLL_LONGTERM may be used in various combinations with each
• • other. Here is what they mean, and how to use them:
  • FOLL_LONGTERM indicates that the page will be held for an indefinite time

• • period _often_ under userspace control. This is contrasted with
• • iov_iter_get_pages() where usages which are transient.

• • period _often_ under userspace control. This is in contrast to
• • iov_iter_get_pages(), where usages which are transient.
  • FIXME: For pages which are part of a filesystem, mappings are subject to the
  • lifetime enforced by the filesystem and we need guarantees that longterm

@@ -2615,11 +2635,32 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,

• Currently only get_user_pages() and get_user_pages_fast() support this flag
• and calls to get_user_pages_[un]locked are specifically not allowed. This
• is due to an incompatibility with the FS DAX check and

• • FAULT_FLAG_ALLOW_RETRY

• • FAULT_FLAG_ALLOW_RETRY.

• • In the CMA case: longterm pins in a CMA region would unnecessarily fragment
• • that region. And so CMA attempts to migrate the page before pinning when

• • In the CMA case: long term pins in a CMA region would unnecessarily fragment
• • that region. And so, CMA attempts to migrate the page before pinning, when
  • FOLL_LONGTERM is specified.
• • FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
• • but an additional pin counting system) will be invoked. This is intended for
• • anything that gets a page reference and then touches page data (for example,
• • Direct IO). This lets the filesystem know that some non-file-system entity is
• • potentially changing the pages' data. In contrast to FOLL_GET (whose pages
• • are released via put_page()), FOLL_PIN pages must be released, ultimately, by
• • a call to put_user_page().
• • FOLL_PIN is similar to FOLL_GET: both of these pin pages. They use different
• • and separate refcounting mechanisms, however, and that means that each has
• • its own acquire and release mechanisms:

• • FOLL_GET: get_user_pages*() to acquire, and put_page() to release.
    

• • FOLL_PIN: pin_user_pages*() or pin_longterm_pages*() to acquire, and
    

• •           put_user_pages to release.
    

• • FOLL_PIN and FOLL_GET are mutually exclusive.

You mean the flags are mutually exclusive for any single call, correct? Because my first thought was that you meant that a page which was pin'ed can't be "got". Which I don't think is true or necessary...

• • Please see Documentation/vm/pin_user_pages.rst for more information.

NIT: I think we should include this file as part of this patch...

*/

static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) diff --git a/mm/gup.c b/mm/gup.c index 8fb0d9cdfaf5..8694bc7b3df3 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -179,6 +179,10 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, spinlock_t *ptl; pte_t *ptep, pte;

• /* FOLL_GET and FOLL_PIN are mutually exclusive. */
• if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==

• 	 (FOLL_PIN | FOLL_GET)))
  

• return ERR_PTR(-EINVAL);
  

retry: if (unlikely(pmd_bad(*pmd))) return no_page_table(vma, flags); @@ -790,7 +794,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,

start = untagged_addr(start);

• VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));

• VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));

  /*

  • If FOLL_FORCE is set then do not force a full fault as the hinting

@@ -1014,7 +1018,16 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, BUG_ON(*locked != 1); }

• if (pages)

• /*

• * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
  

• * is to set FOLL_GET if the caller wants pages[] filled in (but has
  

• * carelessly failed to specify FOLL_GET), so keep doing that, but only
  

• * for FOLL_GET, not for the newer FOLL_PIN.
  

• *
  

• * FOLL_PIN always expects pages to be non-null, but no need to assert
  

• * that here, as any failures will be obvious enough.
  

• */
  

• if (pages && !(flags & FOLL_PIN)) flags |= FOLL_GET;

  pages_done = 0;

@@ -1133,6 +1146,12 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,

• is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
• be called after the page is finished with, and before put_page is called.

• • A note on gup_flags: FOLL_PIN must only be set internally by the
• • pin_user_page*() and pin_longterm_*() APIs, never directly by the caller.
• • That's in order to help avoid mismatches when releasing pages:
• • get_user_pages*() pages must be released via put_page(), while
• • pin_user_pages*() pages must be released via put_user_page().
• • get_user_pages is typically used for fewer-copy IO operations, to get a
  • handle on the memory by some means other than accesses via the user virtual
  • addresses. The pages may be submitted for DMA to devices or accessed via

@@ -1151,6 +1170,14 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas, int *locked) {

• /*

• * As detailed above, FOLL_PIN must only be set internally by the
  

• * pin_user_page*() and pin_longterm_*() APIs, never directly by the
  

• * caller, so enforce that with an assertion:
  

• */
  

• if (WARN_ON_ONCE(gup_flags & FOLL_PIN))

• return -EINVAL;
  

• /*
  • FIXME: Current FOLL_LONGTERM behavior is incompatible with
  • FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on

@@ -1603,11 +1630,25 @@ static __always_inline long __gup_longterm_locked(struct task_struct *tsk,

• and mm being operated on are the current task's and don't allow
• passing of a locked parameter. We also obviously don't pass
• FOLL_REMOTE in here.

• • A note on gup_flags: FOLL_PIN should only be set internally by the
• • pin_user_page*() and pin_longterm_*() APIs, never directly by the caller.
• • That's in order to help avoid mismatches when releasing pages:
• • get_user_pages*() pages must be released via put_page(), while
• • pin_user_pages*() pages must be released via put_user_page().

Rather than put this here should we put it next to the definition of FOLL_PIN? Because now we have this text 2x... :-/

*/ long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas) {

• /*

• * As detailed above, FOLL_PIN must only be set internally by the
  

• * pin_user_page*() and pin_longterm_*() APIs, never directly by the
  

• * caller, so enforce that with an assertion:
  

• */
  

• if (WARN_ON_ONCE(gup_flags & FOLL_PIN))

• return -EINVAL;
  

• return __gup_longterm_locked(current, current->mm, start, nr_pages, pages, vmas, gup_flags | FOLL_TOUCH);

} @@ -2366,24 +2407,9 @@ static int __gup_longterm_unlocked(unsigned long start, int nr_pages, return ret; }

-/**

• • get_user_pages_fast() - pin user pages in memory
• • @start: starting user address
• • @nr_pages: number of pages from start to pin
• • @gup_flags: flags modifying pin behaviour
• • @pages: array that receives pointers to the pages pinned.

• • Should be at least nr_pages long.
    

• • Attempt to pin user pages in memory without taking mm->mmap_sem.
• • If not successful, it will fall back to taking the lock and
• • calling get_user_pages().
• • Returns number of pages pinned. This may be fewer than the number
• • requested. If nr_pages is 0 or negative, returns 0. If no pages
• • were pinned, returns -errno.
• */

-int get_user_pages_fast(unsigned long start, int nr_pages,

• 	unsigned int gup_flags, struct page **pages)
  

+static int internal_get_user_pages_fast(unsigned long start, int nr_pages,

• 			unsigned int gup_flags,
  

• 			struct page **pages)
  

{ unsigned long addr, len, end; int nr = 0, ret = 0; @@ -2428,4 +2454,222 @@ int get_user_pages_fast(unsigned long start, int nr_pages,

return ret; }

+/**

• • get_user_pages_fast() - pin user pages in memory
• • @start: starting user address
• • @nr_pages: number of pages from start to pin
• • @gup_flags: flags modifying pin behaviour
• • @pages: array that receives pointers to the pages pinned.

• • Should be at least nr_pages long.
    

• • Attempt to pin user pages in memory without taking mm->mmap_sem.
• • If not successful, it will fall back to taking the lock and
• • calling get_user_pages().
• • A note on gup_flags: FOLL_PIN must only be set internally by the
• • pin_user_page*() and pin_longterm_*() APIs, never directly by the caller.
• • That's in order to help avoid mismatches when releasing pages:
• • get_user_pages*() pages must be released via put_page(), while
• • pin_user_pages*() pages must be released via put_user_page().
• • Returns number of pages pinned. This may be fewer than the number requested.
• • If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
• • -errno.
• */

+int get_user_pages_fast(unsigned long start, int nr_pages,

• 	unsigned int gup_flags, struct page **pages)
  

+{

• /*

• * As detailed above, FOLL_PIN must only be set internally by the
  

• * pin_user_page*() and pin_longterm_*() APIs, never directly by the
  

• * caller, so enforce that:
  

• */
  

• if (WARN_ON_ONCE(gup_flags & FOLL_PIN))

• return -EINVAL;
  

• return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);

+} EXPORT_SYMBOL_GPL(get_user_pages_fast);

+/**

• • pin_user_pages_fast() - pin user pages in memory without taking locks
• • Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See
• • get_user_pages_fast() for documentation on the function arguments, because
• • the arguments here are identical.
• • FOLL_PIN means that the pages must be released via put_user_page(). Please
• • see Documentation/vm/pin_user_pages.rst for further details.
• • This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
• • is NOT intended for Case 2 (RDMA: long-term pins).
• */

+int pin_user_pages_fast(unsigned long start, int nr_pages,

• 	unsigned int gup_flags, struct page **pages)
  

+{

• /* FOLL_GET and FOLL_PIN are mutually exclusive. */
• if (WARN_ON_ONCE(gup_flags & FOLL_GET))

• return -EINVAL;
  

• gup_flags |= FOLL_PIN;
• return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);

+} +EXPORT_SYMBOL_GPL(pin_user_pages_fast);

+/**

• • pin_longterm_pages_fast() - pin user pages in memory without taking locks
• • Nearly the same as get_user_pages_fast(), except that FOLL_PIN and
• • FOLL_LONGTERM are set. See get_user_pages_fast() for documentation on the
• • function arguments, because the arguments here are identical.
• • FOLL_PIN means that the pages must be released via put_user_page(). Please
• • see Documentation/vm/pin_user_pages.rst for further details.
• • FOLL_LONGTERM means that the pages are being pinned for "long term" use,
• • typically by a non-CPU device, and we cannot be sure that waiting for a
• • pinned page to become unpin will be effective.
• • This is intended for Case 2 (RDMA: long-term pins) of the FOLL_PIN
• • documentation.
• */

+int pin_longterm_pages_fast(unsigned long start, int nr_pages,

• 	    unsigned int gup_flags, struct page **pages)
  

+{

• /* FOLL_GET and FOLL_PIN are mutually exclusive. */
• if (WARN_ON_ONCE(gup_flags & FOLL_GET))

• return -EINVAL;
  

• gup_flags |= (FOLL_PIN | FOLL_LONGTERM);
• return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);

+} +EXPORT_SYMBOL_GPL(pin_longterm_pages_fast);

+/**

• • pin_user_pages_remote() - pin pages for (typically) use by Direct IO, and
• • return the pages to the user.
• • Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See
• • get_user_pages_remote() for documentation on the function arguments, because
• • the arguments here are identical.
• • FOLL_PIN means that the pages must be released via put_user_page(). Please
• • see Documentation/vm/pin_user_pages.rst for details.
• • This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
• • is NOT intended for Case 2 (RDMA: long-term pins).
• */

+long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,

• 	   unsigned long start, unsigned long nr_pages,
  

• 	   unsigned int gup_flags, struct page **pages,
  

• 	   struct vm_area_struct **vmas, int *locked)
  

+{

• /* FOLL_GET and FOLL_PIN are mutually exclusive. */
• if (WARN_ON_ONCE(gup_flags & FOLL_GET))

• return -EINVAL;
  

• gup_flags |= FOLL_TOUCH | FOLL_REMOTE | FOLL_PIN;
• return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,

• 		       locked, gup_flags);
  

+} +EXPORT_SYMBOL(pin_user_pages_remote);

+/**

• • pin_longterm_pages_remote() - pin pages for (typically) use by Direct IO, and
• • return the pages to the user.
• • Nearly the same as get_user_pages_remote(), but note that FOLL_TOUCH is not
• • set, and FOLL_PIN and FOLL_LONGTERM are set. See get_user_pages_remote() for
• • documentation on the function arguments, because the arguments here are
• • identical.
• • FOLL_PIN means that the pages must be released via put_user_page(). Please
• • see Documentation/vm/pin_user_pages.rst for further details.
• • FOLL_LONGTERM means that the pages are being pinned for "long term" use,
• • typically by a non-CPU device, and we cannot be sure that waiting for a
• • pinned page to become unpin will be effective.
• • This is intended for Case 2 (RDMA: long-term pins) in
• • Documentation/vm/pin_user_pages.rst.
• */

+long pin_longterm_pages_remote(struct task_struct *tsk, struct mm_struct *mm,

• 	       unsigned long start, unsigned long nr_pages,
  

• 	       unsigned int gup_flags, struct page **pages,
  

• 	       struct vm_area_struct **vmas, int *locked)
  

+{

• /* FOLL_GET and FOLL_PIN are mutually exclusive. */
• if (WARN_ON_ONCE(gup_flags & FOLL_GET))

• return -EINVAL;
  

• /*

• * FIXME: as noted in the get_user_pages_remote() implementation, it
  

• * is not yet possible to safely set FOLL_LONGTERM here. FOLL_LONGTERM
  

• * needs to be set, but for now the best we can do is a "TODO" item.
  

• */
  

Wait? Why can't we set FOLL_LONGTERM here? pin_* are new calls which are not used yet right?

You set it in the other new pin_* functions?

Ira

• gup_flags |= FOLL_REMOTE | FOLL_PIN;
• return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,

• 		       locked, gup_flags);
  

+} +EXPORT_SYMBOL(pin_longterm_pages_remote);

+/**

• • pin_user_pages() - pin user pages in memory for use by other devices
• • Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and
• • FOLL_PIN is set.
• • FOLL_PIN means that the pages must be released via put_user_page(). Please
• • see Documentation/vm/pin_user_pages.rst for details.
• • This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
• • is NOT intended for Case 2 (RDMA: long-term pins).
• */

+long pin_user_pages(unsigned long start, unsigned long nr_pages,

•     unsigned int gup_flags, struct page **pages,
  

•     struct vm_area_struct **vmas)
  

+{

• /* FOLL_GET and FOLL_PIN are mutually exclusive. */
• if (WARN_ON_ONCE(gup_flags & FOLL_GET))

• return -EINVAL;
  

• gup_flags |= FOLL_PIN;
• return __gup_longterm_locked(current, current->mm, start, nr_pages,

• 		     pages, vmas, gup_flags);
  

+} +EXPORT_SYMBOL(pin_user_pages);

+/**

• • pin_longterm_pages() - pin user pages in memory for long-term use (RDMA,
• • typically)
• • Nearly the same as get_user_pages(), except that FOLL_PIN and FOLL_LONGTERM
• • are set. See get_user_pages_fast() for documentation on the function
• • arguments, because the arguments here are identical.
• • FOLL_PIN means that the pages must be released via put_user_page(). Please
• • see Documentation/vm/pin_user_pages.rst for further details.
• • FOLL_LONGTERM means that the pages are being pinned for "long term" use,
• • typically by a non-CPU device, and we cannot be sure that waiting for a
• • pinned page to become unpin will be effective.
• • This is intended for Case 2 (RDMA: long-term pins) in
• • Documentation/vm/pin_user_pages.rst.
• */

+long pin_longterm_pages(unsigned long start, unsigned long nr_pages,

• 	unsigned int gup_flags, struct page **pages,
  

• 	struct vm_area_struct **vmas)
  

+{

• /* FOLL_GET and FOLL_PIN are mutually exclusive. */
• if (WARN_ON_ONCE(gup_flags & FOLL_GET))

• return -EINVAL;
  

• gup_flags |= FOLL_PIN | FOLL_LONGTERM;
• return __gup_longterm_locked(current, current->mm, start, nr_pages,

• 		     pages, vmas, gup_flags);
  

+}

+EXPORT_SYMBOL(pin_longterm_pages);

2.23.0

On Wed, Oct 30, 2019 at 03:49:18PM -0700, John Hubbard wrote:

Convert infiniband to use the new wrapper calls, and stop explicitly setting FOLL_LONGTERM at the call sites.

The new pin_longterm_*() calls replace get_user_pages*() calls, and set both FOLL_LONGTERM and a new FOLL_PIN flag. The FOLL_PIN flag requires that the caller must return the pages via put_user_page*() calls, but infiniband was already doing that as part of an earlier commit.

NOTE: I'm not 100% convinced that mixing the flags and new calls like this is good. I think we are going to need a lot more documentation on which flags are "user" accessible vs not...

Reviewed-by: Ira Weiny ira.weiny@intel.com

Signed-off-by: John Hubbard jhubbard@nvidia.com

drivers/infiniband/core/umem.c | 5 ++--- drivers/infiniband/core/umem_odp.c | 10 +++++----- drivers/infiniband/hw/hfi1/user_pages.c | 4 ++-- drivers/infiniband/hw/mthca/mthca_memfree.c | 3 +-- drivers/infiniband/hw/qib/qib_user_pages.c | 8 ++++---- drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +- drivers/infiniband/hw/usnic/usnic_uiom.c | 9 ++++----- drivers/infiniband/sw/siw/siw_mem.c | 5 ++--- 8 files changed, 21 insertions(+), 25 deletions(-)

diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 24244a2f68cc..c5a78d3e674b 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -272,11 +272,10 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,

while (npages) { down_read(&mm->mmap_sem);

• ret = get_user_pages(cur_base,
  

• ret = pin_longterm_pages(cur_base,
  	     min_t(unsigned long, npages,
  		   PAGE_SIZE / sizeof (struct page *)),
  

• 		     gup_flags | FOLL_LONGTERM,
  

• 		     page_list, NULL);
  

• 		     gup_flags, page_list, NULL);
  

  if (ret < 0) { up_read(&mm->mmap_sem); goto umem_release;

diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index 163ff7ba92b7..a38b67b83db5 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -534,7 +534,7 @@ static int ib_umem_odp_map_dma_single_page( } else if (umem_odp->page_list[page_index] == page) { umem_odp->dma_list[page_index] |= access_mask; } else {

• pr_err("error: got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n",
  

• pr_err("error: got different pages in IB device and from pin_longterm_pages. IB device page: %p, gup page: %p\n",
    umem_odp->page_list[page_index], page);
  

  /* Better remove the mapping now, to prevent any further
  • damage. */

@@ -639,11 +639,11 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, /* * Note: this might result in redundent page getting. We can * avoid this by checking dma_list to be 0 before calling

•  * get_user_pages. However, this make the code much more
  

•  * complex (and doesn't gain us much performance in most use
  

•  * cases).
  

•  * pin_longterm_pages. However, this makes the code much
  

•  * more complex (and doesn't gain us much performance in most
  

•  * use cases).
  

  */

• npages = get_user_pages_remote(owning_process, owning_mm,
  

• npages = pin_longterm_pages_remote(owning_process, owning_mm,
  	user_virt, gup_num_pages,
  	flags, local_page_list, NULL, NULL);
  

  up_read(&owning_mm->mmap_sem);

diff --git a/drivers/infiniband/hw/hfi1/user_pages.c b/drivers/infiniband/hw/hfi1/user_pages.c index 469acb961fbd..9b55b0a73e29 100644 --- a/drivers/infiniband/hw/hfi1/user_pages.c +++ b/drivers/infiniband/hw/hfi1/user_pages.c @@ -104,9 +104,9 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np bool writable, struct page **pages) { int ret;

• unsigned int gup_flags = FOLL_LONGTERM | (writable ? FOLL_WRITE : 0);

• unsigned int gup_flags = (writable ? FOLL_WRITE : 0);

• ret = get_user_pages_fast(vaddr, npages, gup_flags, pages);

• ret = pin_longterm_pages_fast(vaddr, npages, gup_flags, pages); if (ret < 0) return ret;

diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c index edccfd6e178f..beec7e4b8a96 100644 --- a/drivers/infiniband/hw/mthca/mthca_memfree.c +++ b/drivers/infiniband/hw/mthca/mthca_memfree.c @@ -472,8 +472,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, goto out; }

• ret = get_user_pages_fast(uaddr & PAGE_MASK, 1,

• 		  FOLL_WRITE | FOLL_LONGTERM, pages);
  

• ret = pin_longterm_pages_fast(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages); if (ret < 0) goto out;

diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c index 6bf764e41891..684a14e14d9b 100644 --- a/drivers/infiniband/hw/qib/qib_user_pages.c +++ b/drivers/infiniband/hw/qib/qib_user_pages.c @@ -108,10 +108,10 @@ int qib_get_user_pages(unsigned long start_page, size_t num_pages,

down_read(&current->mm->mmap_sem); for (got = 0; got < num_pages; got += ret) {

• ret = get_user_pages(start_page + got * PAGE_SIZE,
  

• 		     num_pages - got,
  

• 		     FOLL_LONGTERM | FOLL_WRITE | FOLL_FORCE,
  

• 		     p + got, NULL);
  

• ret = pin_longterm_pages(start_page + got * PAGE_SIZE,
  

• 			 num_pages - got,
  

• 			 FOLL_WRITE | FOLL_FORCE,
  

• 			 p + got, NULL);
  

  if (ret < 0) { up_read(&current->mm->mmap_sem); goto bail_release;

diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c index 05190edc2611..fd86a9d19370 100644 --- a/drivers/infiniband/hw/qib/qib_user_sdma.c +++ b/drivers/infiniband/hw/qib/qib_user_sdma.c @@ -670,7 +670,7 @@ static int qib_user_sdma_pin_pages(const struct qib_devdata *dd, else j = npages;

• ret = get_user_pages_fast(addr, j, FOLL_LONGTERM, pages);
  

• ret = pin_longterm_pages_fast(addr, j, 0, pages);
  

  if (ret != j) { i = 0; j = ret;

diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c index 62e6ffa9ad78..6b90ca1c3771 100644 --- a/drivers/infiniband/hw/usnic/usnic_uiom.c +++ b/drivers/infiniband/hw/usnic/usnic_uiom.c @@ -141,11 +141,10 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable, ret = 0;

while (npages) {

• ret = get_user_pages(cur_base,
  

• 		     min_t(unsigned long, npages,
  

• 		     PAGE_SIZE / sizeof(struct page *)),
  

• 		     gup_flags | FOLL_LONGTERM,
  

• 		     page_list, NULL);
  

• ret = pin_longterm_pages(cur_base,
  

• 			 min_t(unsigned long, npages,
  

• 			     PAGE_SIZE / sizeof(struct page *)),
  

• 			 gup_flags, page_list, NULL);
  
  

  if (ret < 0) goto out;

diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c index e99983f07663..20e663d7ada8 100644 --- a/drivers/infiniband/sw/siw/siw_mem.c +++ b/drivers/infiniband/sw/siw/siw_mem.c @@ -426,9 +426,8 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable) while (nents) { struct page **plist = &umem->page_chunk[i].plist[got];

• 	rv = get_user_pages(first_page_va, nents,
  

• 			    foll_flags | FOLL_LONGTERM,
  

• 			    plist, NULL);
  

• 	rv = pin_longterm_pages(first_page_va, nents,
  

• 				foll_flags, plist, NULL);
  if (rv < 0)
  	goto out_sem_up;
  
  

-- 2.23.0

On 10/31/19 4:35 PM, Ira Weiny wrote:

On Wed, Oct 30, 2019 at 03:49:19PM -0700, John Hubbard wrote:

...

Convert process_vm_access to use the new pin_user_pages_remote() call, which sets FOLL_PIN. Setting FOLL_PIN is now required for code that requires tracking of pinned pages.

Also, release the pages via put_user_page*().

Also, rename "pages" to "pinned_pages", as this makes for easier reading of process_vm_rw_single_vec().

Ok... but it made review a bit harder...

Yes, sorry about that. After dealing with "pages means struct page *[]" for all this time, having an "int pages" just was a step too far for me here. :)

Thanks for working through it.

thanks,

John Hubbard NVIDIA

Reviewed-by: Ira Weiny ira.weiny@intel.com

...

Signed-off-by: John Hubbard jhubbard@nvidia.com

mm/process_vm_access.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index 357aa7bef6c0..fd20ab675b85 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -42,12 +42,11 @@ static int process_vm_rw_pages(struct page **pages, if (copy > len) copy = len;

• if (vm_write) {
  

• if (vm_write)
  copied = copy_page_from_iter(page, offset, copy, iter);
  

• 	set_page_dirty_lock(page);
  

• } else {
  

• else
  copied = copy_page_to_iter(page, offset, copy, iter);
  

• }
  

• len -= copied; if (copied < copy && iov_iter_count(iter)) return -EFAULT;

@@ -96,7 +95,7 @@ static int process_vm_rw_single_vec(unsigned long addr, flags |= FOLL_WRITE;

while (!rc && nr_pages && iov_iter_count(iter)) {

• int pages = min(nr_pages, max_pages_per_loop);
  

• int pinned_pages = min(nr_pages, max_pages_per_loop);
  

  int locked = 1; size_t bytes;

@@ -106,14 +105,15 @@ static int process_vm_rw_single_vec(unsigned long addr, * current/current->mm */ down_read(&mm->mmap_sem);

• pages = get_user_pages_remote(task, mm, pa, pages, flags,
  

• 			      process_pages, NULL, &locked);
  

• pinned_pages = pin_user_pages_remote(task, mm, pa, pinned_pages,
  

• 				     flags, process_pages,
  

• 				     NULL, &locked);
  

  if (locked) up_read(&mm->mmap_sem);

• if (pages <= 0)
  

• if (pinned_pages <= 0)
  return -EFAULT;
  
  

• bytes = pages * PAGE_SIZE - start_offset;
  

• bytes = pinned_pages * PAGE_SIZE - start_offset;
  

  if (bytes > len) bytes = len;

@@ -122,10 +122,12 @@ static int process_vm_rw_single_vec(unsigned long addr, vm_write); len -= bytes; start_offset = 0;

• nr_pages -= pages;
  

• pa += pages * PAGE_SIZE;
  

• while (pages)
  

• 	put_page(process_pages[--pages]);
  

• nr_pages -= pinned_pages;
  

• pa += pinned_pages * PAGE_SIZE;
  

• /* If vm_write is set, the pages need to be made dirty: */
  

• put_user_pages_dirty_lock(process_pages, pinned_pages,
  

• 			  vm_write);
  

  }

  return rc;

-- 2.23.0

On Wed, Oct 30, 2019 at 03:49:20PM -0700, John Hubbard wrote:

Convert drm/via to use the new pin_user_pages_fast() call, which sets FOLL_PIN. Setting FOLL_PIN is now required for code that requires tracking of pinned pages, and therefore for any code that calls put_user_page().

Reviewed-by: Ira Weiny ira.weiny@intel.com

Signed-off-by: John Hubbard jhubbard@nvidia.com

drivers/gpu/drm/via/via_dmablit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/via/via_dmablit.c b/drivers/gpu/drm/via/via_dmablit.c index 3db000aacd26..37c5e572993a 100644 --- a/drivers/gpu/drm/via/via_dmablit.c +++ b/drivers/gpu/drm/via/via_dmablit.c @@ -239,7 +239,7 @@ via_lock_all_dma_pages(drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer) vsg->pages = vzalloc(array_size(sizeof(struct page *), vsg->num_pages)); if (NULL == vsg->pages) return -ENOMEM;

• ret = get_user_pages_fast((unsigned long)xfer->mem_addr,

• ret = pin_user_pages_fast((unsigned long)xfer->mem_addr, vsg->num_pages, vsg->direction == DMA_FROM_DEVICE ? FOLL_WRITE : 0, vsg->pages);

-- 2.23.0

On Wed, Oct 30, 2019 at 03:49:21PM -0700, John Hubbard wrote:

Convert fs/io_uring to use the new pin_user_pages() call, which sets FOLL_PIN. Setting FOLL_PIN is now required for code that requires tracking of pinned pages, and therefore for any code that calls put_user_page().

Reviewed-by: Ira Weiny ira.weiny@intel.com

Signed-off-by: John Hubbard jhubbard@nvidia.com

fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c index a30c4f622cb3..d3924b1760eb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3431,9 +3431,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,

ret = 0;
down_read(&current->mm->mmap_sem);

• pret = get_user_pages(ubuf, nr_pages,
  

• 		      FOLL_WRITE | FOLL_LONGTERM,
  

• 		      pages, vmas);
  

• pret = pin_longterm_pages(ubuf, nr_pages, FOLL_WRITE, pages,
  

• 			  vmas);
  

  if (pret == nr_pages) { /* don't support file backed memory */ for (j = 0; j < nr_pages; j++) {

-- 2.23.0

On Wed, Oct 30, 2019 at 03:49:22PM -0700, John Hubbard wrote:

Convert net/xdp to use the new pin_longterm_pages() call, which sets FOLL_PIN. Setting FOLL_PIN is now required for code that requires tracking of pinned pages.

Reviewed-by: Ira Weiny ira.weiny@intel.com

Signed-off-by: John Hubbard jhubbard@nvidia.com

net/xdp/xdp_umem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 16d5f353163a..4d56dfb1139a 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -285,8 +285,8 @@ static int xdp_umem_pin_pages(struct xdp_umem *umem) return -ENOMEM;

down_read(&current->mm->mmap_sem);

• npgs = get_user_pages(umem->address, umem->npgs,

• 	      gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL);
  

• npgs = pin_longterm_pages(umem->address, umem->npgs, gup_flags,

• 		  &umem->pgs[0], NULL);
  

  up_read(&current->mm->mmap_sem);

  if (npgs != umem->npgs) {

-- 2.23.0

On Wed, Oct 30, 2019 at 03:49:24PM -0700, John Hubbard wrote:

1. Change v4l2 from get_user_pages(FOLL_LONGTERM), to

pin_longterm_pages(), which sets both FOLL_LONGTERM and FOLL_PIN.

2. Because all FOLL_PIN-acquired pages must be released via

put_user_page(), also convert the put_page() call over to put_user_pages_dirty_lock().

Reviewed-by: Ira Weiny ira.weiny@intel.com

Cc: Mauro Carvalho Chehab mchehab@kernel.org Signed-off-by: John Hubbard jhubbard@nvidia.com

---

drivers/media/v4l2-core/videobuf-dma-sg.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c index 28262190c3ab..9b9c5b37bf59 100644 --- a/drivers/media/v4l2-core/videobuf-dma-sg.c +++ b/drivers/media/v4l2-core/videobuf-dma-sg.c @@ -183,12 +183,12 @@ static int videobuf_dma_init_user_locked(struct videobuf_dmabuf *dma, dprintk(1, "init user [0x%lx+0x%lx => %d pages]\n", data, size, dma->nr_pages);

• err = get_user_pages(data & PAGE_MASK, dma->nr_pages,

• 	     flags | FOLL_LONGTERM, dma->pages, NULL);
  

• err = pin_longterm_pages(data & PAGE_MASK, dma->nr_pages,

• 		 flags, dma->pages, NULL);
  
  

  if (err != dma->nr_pages) { dma->nr_pages = (err >= 0) ? err : 0;

• dprintk(1, "get_user_pages: err=%d [%d]\n", err,
  

• dprintk(1, "pin_longterm_pages: err=%d [%d]\n", err,
  dma->nr_pages);
  

  return err < 0 ? err : -EINVAL; }

@@ -349,11 +349,8 @@ int videobuf_dma_free(struct videobuf_dmabuf *dma) BUG_ON(dma->sglen);

if (dma->pages) {

• for (i = 0; i < dma->nr_pages; i++) {
  

• 	if (dma->direction == DMA_FROM_DEVICE)
  

• 		set_page_dirty_lock(dma->pages[i]);
  

• 	put_page(dma->pages[i]);
  

• }
  

• put_user_pages_dirty_lock(dma->pages, dma->nr_pages,
  

• 			  dma->direction == DMA_FROM_DEVICE);
  

  kfree(dma->pages); dma->pages = NULL; }

-- 2.23.0

On 10/30/19 3:49 PM, John Hubbard wrote:

This also fixes one or two likely bugs.

Well, actually just one...

1. Change vfio from get_user_pages(FOLL_LONGTERM), to

pin_longterm_pages(), which sets both FOLL_LONGTERM and FOLL_PIN.

Note that this is a change in behavior, because the get_user_pages_remote() call was not setting FOLL_LONGTERM, but the new pin_user_pages_remote() call that replaces it, *is* setting FOLL_LONGTERM. It is important to set FOLL_LONGTERM, because the DMA case requires it. Please see the FOLL_PIN documentation in include/linux/mm.h, and Documentation/pin_user_pages.rst for details.

Correction: the above comment is stale and wrong. I wrote it before getting further into the details, and the patch doesn't do this.

Instead, it keeps exactly the old behavior: pin_longterm_pages_remote() is careful to avoid setting FOLL_LONGTERM. Instead of setting that flag, it drops in a "TODO" comment nearby. :)

I'll update the commit description in the next version of the series.

thanks,

John Hubbard NVIDIA

2. Because all FOLL_PIN-acquired pages must be released via

put_user_page(), also convert the put_page() call over to put_user_pages().

Note that this effectively changes the code's behavior in vfio_iommu_type1.c: put_pfn(): it now ultimately calls set_page_dirty_lock(), instead of set_page_dirty(). This is probably more accurate.

As Christoph Hellwig put it, "set_page_dirty() is only safe if we are dealing with a file backed page where we have reference on the inode it hangs off." [1]

[1] https://lore.kernel.org/r/20190723153640.GB720@lst.de

Cc: Alex Williamson alex.williamson@redhat.com Signed-off-by: John Hubbard jhubbard@nvidia.com

---

drivers/vfio/vfio_iommu_type1.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index d864277ea16f..795e13f3ef08 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -327,9 +327,8 @@ static int put_pfn(unsigned long pfn, int prot) { if (!is_invalid_reserved_pfn(pfn)) { struct page *page = pfn_to_page(pfn);

• if (prot & IOMMU_WRITE)
  

• 	SetPageDirty(page);
  

• put_page(page);
  

• put_user_pages_dirty_lock(&page, 1, prot & IOMMU_WRITE);
  

  return 1; } return 0;

@@ -349,11 +348,11 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,

down_read(&mm->mmap_sem); if (mm == current->mm) {

• ret = get_user_pages(vaddr, 1, flags | FOLL_LONGTERM, page,
  

• 		     vmas);
  

• ret = pin_longterm_pages(vaddr, 1, flags, page, vmas);
  

  } else {

• ret = get_user_pages_remote(NULL, mm, vaddr, 1, flags, page,
  

• 			    vmas, NULL);
  

• ret = pin_longterm_pages_remote(NULL, mm, vaddr, 1,
  

• 				flags, page, vmas,
  

• 				NULL);
  

  /*
  • The lifetime of a vaddr_get_pfn() page pin is
  • userspace-controlled. In the fs-dax case this could

@@ -363,7 +362,7 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, */ if (ret > 0 && vma_is_fsdax(vmas[0])) { ret = -EOPNOTSUPP;

• 	put_page(page[0]);
  

• 	put_user_page(page[0]);
  

  } } up_read(&mm->mmap_sem);

On Wed, Oct 30, 2019 at 03:49:30PM -0700, John Hubbard wrote:

Document the new pin_user_pages() and related calls and behavior.

Thanks to Jan Kara and Vlastimil Babka for explaining the 4 cases in this documentation. (I've reworded it and expanded on it slightly.)

As I said before I think this may be better in a previous patch where you reference it.

Ira

Cc: Jonathan Corbet corbet@lwn.net Signed-off-by: John Hubbard jhubbard@nvidia.com

---

Documentation/vm/index.rst | 1 + Documentation/vm/pin_user_pages.rst | 213 ++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 Documentation/vm/pin_user_pages.rst

diff --git a/Documentation/vm/index.rst b/Documentation/vm/index.rst index e8d943b21cf9..7194efa3554a 100644 --- a/Documentation/vm/index.rst +++ b/Documentation/vm/index.rst @@ -44,6 +44,7 @@ descriptions of data structures and algorithms. page_migration page_frags page_owner

• pin_user_pages remap_file_pages slub split_page_table_lock

diff --git a/Documentation/vm/pin_user_pages.rst b/Documentation/vm/pin_user_pages.rst new file mode 100644 index 000000000000..7110bca3f188 --- /dev/null +++ b/Documentation/vm/pin_user_pages.rst @@ -0,0 +1,213 @@ +.. SPDX-License-Identifier: GPL-2.0

+==================================================== +pin_user_pages() and related calls +====================================================

+.. contents:: :local:

+Overview +========

+This document describes the following functions: ::

• pin_user_pages
• pin_user_pages_fast
• pin_user_pages_remote
• pin_longterm_pages
• pin_longterm_pages_fast
• pin_longterm_pages_remote

+Basic description of FOLL_PIN +=============================

+A new flag for get_user_pages ("gup") has been added: FOLL_PIN. FOLL_PIN has +significant interactions and interdependencies with FOLL_LONGTERM, so both are +covered here.

+Both FOLL_PIN and FOLL_LONGTERM are "internal" to gup, meaning that neither +FOLL_PIN nor FOLL_LONGTERM should not appear at the gup call sites. This allows +the associated wrapper functions (pin_user_pages and others) to set the correct +combination of these flags, and to check for problems as well.

+FOLL_PIN and FOLL_GET are mutually exclusive for a given gup call. However, +multiple threads and call sites are free to pin the same struct pages, via both +FOLL_PIN and FOLL_GET. It's just the call site that needs to choose one or the +other, not the struct page(s).

+The FOLL_PIN implementation is nearly the same as FOLL_GET, except that FOLL_PIN +uses a different reference counting technique.

+FOLL_PIN is a prerequisite to FOLL_LONGTGERM. Another way of saying that is, +FOLL_LONGTERM is a specific case, more restrictive case of FOLL_PIN.

+Which flags are set by each wrapper +===================================

+Only FOLL_PIN and FOLL_LONGTERM are covered here. These flags are added to +whatever flags the caller provides::

• Function gup flags (FOLL_PIN or FOLL_LONGTERM only)

• ---

• pin_user_pages FOLL_PIN
• pin_user_pages_fast FOLL_PIN
• pin_user_pages_remote FOLL_PIN
• pin_longterm_pages FOLL_PIN | FOLL_LONGTERM
• pin_longterm_pages_fast FOLL_PIN | FOLL_LONGTERM
• pin_longterm_pages_remote FOLL_PIN | FOLL_LONGTERM

+Tracking dma-pinned pages +=========================

+Some of the key design constraints, and solutions, for tracking dma-pinned +pages:

+* An actual reference count, per struct page, is required. This is because

• multiple processes may pin and unpin a page.

+* False positives (reporting that a page is dma-pinned, when in fact it is not)

• are acceptable, but false negatives are not.

+* struct page may not be increased in size for this, and all fields are already

• used.

+* Given the above, we can overload the page->_refcount field by using, sort of,

• the upper bits in that field for a dma-pinned count. "Sort of", means that,
• rather than dividing page->_refcount into bit fields, we simple add a medium-
• large value (GUP_PIN_COUNTING_BIAS, initially chosen to be 1024: 10 bits) to
• page->_refcount. This provides fuzzy behavior: if a page has get_page() called
• on it 1024 times, then it will appear to have a single dma-pinned count.
• And again, that's acceptable.

+This also leads to limitations: there are only 32-10==22 bits available for a +counter that increments 10 bits at a time.

+TODO: for 1GB and larger huge pages, this is cutting it close. That's because +when pin_user_pages() follows such pages, it increments the head page by "1" +(where "1" used to mean "+1" for get_user_pages(), but now means "+1024" for +pin_user_pages()) for each tail page. So if you have a 1GB huge page:

+* There are 256K (18 bits) worth of 4 KB tail pages. +* There are 22 bits available to count up via GUP_PIN_COUNTING_BIAS (that is,

• 10 bits at a time)

+* There are 22 - 18 == 4 bits available to count. Except that there aren't,

• because you need to allow for a few normal get_page() calls on the head page,
• as well. Fortunately, the approach of using addition, rather than "hard"
• bitfields, within page->_refcount, allows for sharing these bits gracefully.
• But we're still looking at about 16 references.

+This, however, is a missing feature more than anything else, because it's easily +solved by addressing an obvious inefficiency in the original get_user_pages() +approach of retrieving pages: stop treating all the pages as if they were +PAGE_SIZE. Retrieve huge pages as huge pages. The callers need to be aware of +this, so some work is required. Once that's in place, this limitation mostly +disappears from view, because there will be ample refcounting range available.

+* Callers must specifically request "dma-pinned tracking of pages". In other

• words, just calling get_user_pages() will not suffice; a new set of functions,
• pin_user_page() and related, must be used.

+FOLL_PIN, FOLL_GET, FOLL_LONGTERM: when to use which flags +==========================================================

+Thanks to Jan Kara, Vlastimil Babka and several other -mm people, for describing +these categories:

+CASE 1: Direct IO (DIO) +----------------------- +There are GUP references to pages that are serving +as DIO buffers. These buffers are needed for a relatively short time (so they +are not "long term"). No special synchronization with page_mkclean() or +munmap() is provided. Therefore, flags to set at the call site are: ::

• FOLL_PIN

+...but rather than setting FOLL_PIN directly, call sites should use one of +the pin_user_pages*() routines that set FOLL_PIN.

+CASE 2: RDMA +------------ +There are GUP references to pages that are serving as DMA +buffers. These buffers are needed for a long time ("long term"). No special +synchronization with page_mkclean() or munmap() is provided. Therefore, flags +to set at the call site are: ::

• FOLL_PIN | FOLL_LONGTERM

+TODO: There is also a special case when the pages are DAX pages: in addition to +the above flags, the caller needs something like a layout lease on the +associated file. This is yet to be implemented. When it is implemented, it's +expected that the lease will be a prerequisite to setting FOLL_LONGTERM.

For now we probably want to leave this note out until we figure out how this is going to work. Best to say something like:

Some pages, such as DAX pages, can't be pinned with longterm pins and will fail.

Ira

+CASE 3: ODP +----------- +(Mellanox/Infiniband On Demand Paging: the hardware supports +replayable page faulting). There are GUP references to pages serving as DMA +buffers. For ODP, MMU notifiers are used to synchronize with page_mkclean() +and munmap(). Therefore, normal GUP calls are sufficient, so neither flag +needs to be set.

+CASE 4: Pinning for struct page manipulation only +------------------------------------------------- +Here, normal GUP calls are sufficient, so neither flag needs to be set.

+page_dma_pinned(): the whole point of pinning +=============================================

+The whole point of marking pages as "DMA-pinned" or "gup-pinned" is to be able +to query, "is this page DMA-pinned?" That allows code such as page_mkclean() +(and file system writeback code in general) to make informed decisions about +what to do when a page cannot be unmapped due to such pins.

+What to do in those cases is the subject of a years-long series of discussions +and debates (see the References at the end of this document). It's a TODO item +here: fill in the details once that's worked out. Meanwhile, it's safe to say +that having this available: ::

•    static inline bool page_dma_pinned(struct page *page)
  

+...is a prerequisite to solving the long-running gup+DMA problem.

+Another way of thinking about FOLL_GET, FOLL_PIN, and FOLL_LONGTERM +===================================================================

+Another way of thinking about these flags is as a progression of restrictions: +FOLL_GET is for struct page manipulation, without affecting the data that the +struct page refers to. FOLL_PIN is a *replacement* for FOLL_GET, and is for +short term pins on pages whose data *will* get accessed. As such, FOLL_PIN is +a "more severe" form of pinning. And finally, FOLL_LONGTERM is an even more +restrictive case that has FOLL_PIN as a prerequisite: this is for pages that +will be pinned longterm, and whose data will be accessed.

+Unit testing +============ +This file::

• tools/testing/selftests/vm/gup_benchmark.c

+has the following new calls to exercise the new pin*() wrapper functions:

+* PIN_FAST_BENCHMARK (./gup_benchmark -a) +* PIN_LONGTERM_BENCHMARK (./gup_benchmark -a) +* PIN_BENCHMARK (./gup_benchmark -a)

+You can monitor how many total dma-pinned pages have been acquired and released +since the system was booted, via two new /proc/vmstat entries: ::

• /proc/vmstat/nr_foll_pin_requested
• /proc/vmstat/nr_foll_pin_requested

+Those are both going to show zero, unless CONFIG_DEBUG_VM is set. This is +because there is a noticeable performance drop in put_user_page(), when they +are activated.

+References +==========

+* `Some slow progress on get_user_pages() (Apr 2, 2019) https://lwn.net/Articles/784574/`_ +* `DMA and get_user_pages() (LPC: Dec 12, 2018) https://lwn.net/Articles/774411/`_ +* `The trouble with get_user_pages() (Apr 30, 2018) https://lwn.net/Articles/753027/`_

+John Hubbard, October, 2019

2.23.0