From: CQ Tang cq.tang@intel.com
When an object is pinning, get_pages() is called to allocate memory on a region, if memory pages are not availabe, the region eviction is triggered to find other objects on the same region to be evicted, the selected object is passed to put_pages() call to free the memory pages, before freeing the pages, whether to swap out first to system memory depends on if object is marked as WILLNEED.
After swapped-out, the object is treated as if it does not have any page allocated for it.
Similarly, when an object is pinning, memory pages are allocated from a region, then the object is checked if it had been swapped out before, if yes, swap-in the pages contents into the allocated memory pages.
For this initial swapping code, i915_gem_object_memcpy() is used to copy pages.
Signed-off-by: CQ Tang cq.tang@intel.com --- drivers/gpu/drm/i915/gem/i915_gem_object.c | 12 +- drivers/gpu/drm/i915/gem/i915_gem_object.h | 2 + .../gpu/drm/i915/gem/i915_gem_object_types.h | 6 + drivers/gpu/drm/i915/gem/i915_gem_pages.c | 1 - drivers/gpu/drm/i915/gem/i915_gem_region.c | 139 +++++++++++++++++- drivers/gpu/drm/i915/intel_memory_region.c | 6 + 6 files changed, 162 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 65690e3bf648..7cb5f137522f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -178,6 +178,8 @@ static void __i915_gem_free_object_rcu(struct rcu_head *head) container_of(head, typeof(*obj), rcu); struct drm_i915_private *i915 = to_i915(obj->base.dev);
+ /* Reset shared reservation object */ + obj->base.resv = &obj->base._resv; dma_resv_fini(&obj->base._resv); i915_gem_object_free(obj);
@@ -185,7 +187,7 @@ static void __i915_gem_free_object_rcu(struct rcu_head *head) atomic_dec(&i915->mm.free_count); }
-static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) +void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) { /* Skip serialisation and waking the device if known to be not used. */
@@ -287,6 +289,14 @@ static void i915_gem_free_object(struct drm_gem_object *gem_obj)
GEM_BUG_ON(i915_gem_object_is_framebuffer(obj));
+ /* + * If object had been swapped out, free the hidden object. + */ + if (obj->swapto) { + i915_gem_object_put(obj->swapto); + obj->swapto = NULL; + } + /* * Before we free the object, make sure any pure RCU-only * read-side critical sections are complete, e.g. diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 175258106642..ee1914ed2070 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -366,6 +366,8 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
+void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj); + static inline int __must_check i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 517a606ade8d..e9f42d3137b3 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -316,6 +316,12 @@ struct drm_i915_gem_object {
void *gvt_info; }; + + /** + * object to swap-to if non-null. + */ + bool do_swapping; + struct drm_i915_gem_object *swapto; };
static inline struct drm_i915_gem_object * diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 2cdb7cf63383..d0f3da0925f5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -231,7 +231,6 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) }
__i915_gem_object_reset_page_iter(obj); - obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
return pages; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index e497ff374b13..a437538cd872 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -7,11 +7,135 @@ #include "i915_gem_region.h" #include "i915_drv.h" #include "i915_trace.h" +#include "i915_gem_mman.h" + +static int +i915_gem_object_swapout_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages, unsigned int sizes) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct drm_i915_gem_object *dst, *src; + int err; + + GEM_BUG_ON(obj->swapto); + GEM_BUG_ON(i915_gem_object_has_pages(obj)); + GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); + GEM_BUG_ON(obj->mm.region->type != INTEL_MEMORY_LOCAL); + + assert_object_held(obj); + + /* create a shadow object on smem region */ + dst = i915_gem_object_create_shmem(i915, obj->base.size); + if (IS_ERR(dst)) + return PTR_ERR(dst); + + /* Share the dma-resv between the shadow- and the parent object */ + dst->base.resv = obj->base.resv; + assert_object_held(dst); + + /* + * create working object on the same region as 'obj', + * if 'obj' is used directly, it is set pages and is pinned + * again, other thread may wrongly use 'obj' pages. + */ + src = i915_gem_object_create_region(obj->mm.region, + obj->base.size, 0); + if (IS_ERR(src)) { + i915_gem_object_put(dst); + return PTR_ERR(src); + } + + /* set and pin working object pages */ + i915_gem_object_lock_isolated(src); + __i915_gem_object_set_pages(src, pages, sizes); + __i915_gem_object_pin_pages(src); + + /* copying the pages */ + err = i915_gem_object_memcpy(dst, src); + + __i915_gem_object_unpin_pages(src); + __i915_gem_object_unset_pages(src); + i915_gem_object_unlock(src); + i915_gem_object_put(src); + + if (!err) + obj->swapto = dst; + else + i915_gem_object_put(dst); + + return err; +} + +static int +i915_gem_object_swapin_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages, unsigned int sizes) +{ + struct drm_i915_gem_object *dst, *src; + int err; + + GEM_BUG_ON(!obj->swapto); + GEM_BUG_ON(i915_gem_object_has_pages(obj)); + GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); + GEM_BUG_ON(obj->mm.region->type != INTEL_MEMORY_LOCAL); + + assert_object_held(obj); + + src = obj->swapto; + + /* + * create working object on the same region as 'obj', + * if 'obj' is used directly, it is set pages and is pinned + * again, other thread may wrongly use 'obj' pages. + */ + dst = i915_gem_object_create_region(obj->mm.region, + obj->base.size, 0); + if (IS_ERR(dst)) { + err = PTR_ERR(dst); + return err; + } + + /* @scr is sharing @obj's reservation object */ + assert_object_held(src); + + /* set and pin working object pages */ + i915_gem_object_lock_isolated(dst); + __i915_gem_object_set_pages(dst, pages, sizes); + __i915_gem_object_pin_pages(dst); + + /* copying the pages */ + err = i915_gem_object_memcpy(dst, src); + + __i915_gem_object_unpin_pages(dst); + __i915_gem_object_unset_pages(dst); + i915_gem_object_unlock(dst); + i915_gem_object_put(dst); + + if (!err) { + obj->swapto = NULL; + i915_gem_object_put(src); + } + + return err; +}
void i915_gem_object_put_pages_buddy(struct drm_i915_gem_object *obj, struct sg_table *pages) { + /* if need to save the page contents, swap them out */ + if (obj->do_swapping) { + unsigned int sizes = obj->mm.page_sizes.phys; + + GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); + GEM_BUG_ON(i915_gem_object_is_volatile(obj)); + + if (i915_gem_object_swapout_pages(obj, pages, sizes)) { + /* swapout failed, keep the pages */ + __i915_gem_object_set_pages(obj, pages, sizes); + return; + } + } + __intel_memory_region_put_pages_buddy(obj->mm.region, &obj->mm.blocks);
obj->mm.dirty = false; @@ -95,8 +219,19 @@ i915_gem_object_get_pages_buddy(struct drm_i915_gem_object *obj) sg_mark_end(sg); i915_sg_trim(st);
- /* Intended for kernel internal use only */ - if (obj->flags & I915_BO_ALLOC_CPU_CLEAR) { + /* if we saved the page contents, swap them in */ + if (obj->swapto) { + GEM_BUG_ON(i915_gem_object_is_volatile(obj)); + + ret = i915_gem_object_swapin_pages(obj, st, + sg_page_sizes); + if (ret) { + /* swapin failed, free the pages */ + __intel_memory_region_put_pages_buddy(mem, blocks); + ret = -ENXIO; + goto err_free_sg; + } + } else if (obj->flags & I915_BO_ALLOC_CPU_CLEAR) { struct scatterlist *sg; unsigned long i;
diff --git a/drivers/gpu/drm/i915/intel_memory_region.c b/drivers/gpu/drm/i915/intel_memory_region.c index 185eab497803..afcd6fe6eaff 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.c +++ b/drivers/gpu/drm/i915/intel_memory_region.c @@ -147,6 +147,11 @@ static int intel_memory_region_evict(struct intel_memory_region *mem,
mutex_unlock(&mem->objects.lock);
+ /* tell callee to do swapping */ + if (i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM) + && pass == 1) + obj->do_swapping = true; + if (!i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE)) { if (i915_gem_object_trylock(obj)) { __i915_gem_object_put_pages(obj); @@ -160,6 +165,7 @@ static int intel_memory_region_evict(struct intel_memory_region *mem, } }
+ obj->do_swapping = false; i915_gem_object_put(obj); mutex_lock(&mem->objects.lock);