Device memory that is cache coherent from device and CPU point of view. This is use on platform that have an advance system bus (like CAPI or CCIX). Any page of a process can be migrated to such memory. However, no one should be allow to pin such memory so that it can always be evicted.
Signed-off-by: Alex Sierra alex.sierra@amd.com --- include/linux/memremap.h | 8 ++++++++ include/linux/mm.h | 8 ++++++++ mm/memcontrol.c | 6 +++--- mm/memory-failure.c | 6 +++++- mm/memremap.c | 2 ++ mm/migrate.c | 19 ++++++++++++------- 6 files changed, 38 insertions(+), 11 deletions(-)
diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 77ff5fd0685f..431e1b0bc949 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -39,6 +39,13 @@ struct vmem_altmap { * A more complete discussion of unaddressable memory may be found in * include/linux/hmm.h and Documentation/vm/hmm.rst. * + * MEMORY_DEVICE_PUBLIC: + * Device memory that is cache coherent from device and CPU point of view. This + * is use on platform that have an advance system bus (like CAPI or CCIX). A + * driver can hotplug the device memory using ZONE_DEVICE and with that memory + * type. Any page of a process can be migrated to such memory. However no one + * should be allow to pin such memory so that it can always be evicted. + * * MEMORY_DEVICE_FS_DAX: * Host memory that has similar access semantics as System RAM i.e. DMA * coherent and supports page pinning. In support of coordinating page @@ -59,6 +66,7 @@ struct vmem_altmap { enum memory_type { /* 0 is reserved to catch uninitialized type fields */ MEMORY_DEVICE_PRIVATE = 1, + MEMORY_DEVICE_PUBLIC, MEMORY_DEVICE_FS_DAX, MEMORY_DEVICE_GENERIC, MEMORY_DEVICE_PCI_P2PDMA, diff --git a/include/linux/mm.h b/include/linux/mm.h index e24c904deeec..70a932e8a2ee 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1187,6 +1187,14 @@ static inline bool is_device_private_page(const struct page *page) page->pgmap->type == MEMORY_DEVICE_PRIVATE; }
+static inline bool is_device_page(const struct page *page) +{ + return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) && + is_zone_device_page(page) && + (page->pgmap->type == MEMORY_DEVICE_PRIVATE || + page->pgmap->type == MEMORY_DEVICE_PUBLIC); +} + static inline bool is_pci_p2pdma_page(const struct page *page) { return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) && diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 64ada9e650a5..1599ef1a3b03 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5530,8 +5530,8 @@ static int mem_cgroup_move_account(struct page *page, * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a * target for charge migration. if @target is not NULL, the entry is stored * in target->ent. - * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE - * (so ZONE_DEVICE page and thus not on the lru). + * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PUBLIC + * or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru). * For now we such page is charge like a regular page would be as for all * intent and purposes it is just special memory taking the place of a * regular page. @@ -5565,7 +5565,7 @@ static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma, */ if (page_memcg(page) == mc.from) { ret = MC_TARGET_PAGE; - if (is_device_private_page(page)) + if (is_device_page(page)) ret = MC_TARGET_DEVICE; if (target) target->page = page; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 6f5f78885ab4..16cadbabfc99 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1373,12 +1373,16 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, goto unlock; }
- if (pgmap->type == MEMORY_DEVICE_PRIVATE) { + switch (pgmap->type) { + case MEMORY_DEVICE_PRIVATE: + case MEMORY_DEVICE_PUBLIC: /* * TODO: Handle HMM pages which may need coordination * with device-side memory. */ goto unlock; + default: + break; }
/* diff --git a/mm/memremap.c b/mm/memremap.c index ab949a571e78..685be704b28e 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -294,6 +294,7 @@ void *memremap_pages(struct dev_pagemap *pgmap, int nid)
switch (pgmap->type) { case MEMORY_DEVICE_PRIVATE: + case MEMORY_DEVICE_PUBLIC: if (!IS_ENABLED(CONFIG_DEVICE_PRIVATE)) { WARN(1, "Device private memory not supported\n"); return ERR_PTR(-EINVAL); @@ -493,6 +494,7 @@ void free_zone_device_page(struct page *page) { switch (page->pgmap->type) { case MEMORY_DEVICE_PRIVATE: + case MEMORY_DEVICE_PUBLIC: free_device_page(page); return; case MEMORY_DEVICE_FS_DAX: diff --git a/mm/migrate.c b/mm/migrate.c index e3a10e2a1bb3..7392648966d2 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2565,7 +2565,7 @@ static bool migrate_vma_check_page(struct page *page) * FIXME proper solution is to rework migration_entry_wait() so * it does not need to take a reference on page. */ - return is_device_private_page(page); + return is_device_page(page); }
/* For file back page */ @@ -2854,7 +2854,7 @@ EXPORT_SYMBOL(migrate_vma_setup); * handle_pte_fault() * do_anonymous_page() * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE - * private page. + * private or public page. */ static void migrate_vma_insert_page(struct migrate_vma *migrate, unsigned long addr, @@ -2925,10 +2925,15 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE); entry = swp_entry_to_pte(swp_entry); + } else if (is_device_page(page)) { + entry = pte_mkold(mk_pte(page, + READ_ONCE(vma->vm_page_prot))); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(pte_mkdirty(entry)); } else { /* - * For now we only support migrating to un-addressable - * device memory. + * We support migrating to private and public types + * for device zone memory. */ pr_warn_once("Unsupported ZONE_DEVICE page type.\n"); goto abort; @@ -3034,10 +3039,10 @@ void migrate_vma_pages(struct migrate_vma *migrate) mapping = page_mapping(page);
if (is_zone_device_page(newpage)) { - if (is_device_private_page(newpage)) { + if (is_device_page(newpage)) { /* - * For now only support private anonymous when - * migrating to un-addressable device memory. + * For now only support private and public + * anonymous when migrating to device memory. */ if (mapping) { migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;