Volatile objects are marked as DONTNEED while pinned, therefore once unpinned the backing store can be discarded.
Signed-off-by: Matthew Auld matthew.auld@intel.com Signed-off-by: CQ Tang cq.tang@intel.com Cc: Joonas Lahtinen joonas.lahtinen@linux.intel.com Cc: Abdiel Janulgue abdiel.janulgue@linux.intel.com --- drivers/gpu/drm/i915/gem/i915_gem_internal.c | 17 +++--- drivers/gpu/drm/i915/gem/i915_gem_object.h | 6 ++ .../gpu/drm/i915/gem/i915_gem_object_types.h | 3 +- drivers/gpu/drm/i915/gem/i915_gem_pages.c | 6 ++ drivers/gpu/drm/i915/gem/i915_gem_region.c | 7 ++- .../gpu/drm/i915/gem/selftests/huge_pages.c | 12 ++-- drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 5 +- .../drm/i915/selftests/intel_memory_region.c | 56 +++++++++++++++++++ 8 files changed, 91 insertions(+), 21 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index 0c41e04ab8fa..5e72cb1cc2d3 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -117,13 +117,6 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) goto err; }
- /* Mark the pages as dontneed whilst they are still pinned. As soon - * as they are unpinned they are allowed to be reaped by the shrinker, - * and the caller is expected to repopulate - the contents of this - * object are only valid whilst active and pinned. - */ - obj->mm.madv = I915_MADV_DONTNEED; - __i915_gem_object_set_pages(obj, st, sg_page_sizes);
return 0; @@ -143,7 +136,6 @@ static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, internal_free_pages(pages);
obj->mm.dirty = false; - obj->mm.madv = I915_MADV_WILLNEED; }
static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { @@ -188,6 +180,15 @@ i915_gem_object_create_internal(struct drm_i915_private *i915, drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &i915_gem_object_internal_ops);
+ /* + * Mark the object as volatile, such that the pages are marked as + * dontneed whilst they are still pinned. As soon as they are unpinned + * they are allowed to be reaped by the shrinker, and the caller is + * expected to repopulate - the contents of this object are only valid + * whilst active and pinned. + */ + obj->flags = I915_BO_ALLOC_VOLATILE; + obj->read_domains = I915_GEM_DOMAIN_CPU; obj->write_domain = I915_GEM_DOMAIN_CPU;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 3714cf234d64..1af838050d6c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -122,6 +122,12 @@ i915_gem_object_lock_fence(struct drm_i915_gem_object *obj); void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj, struct dma_fence *fence);
+static inline bool +i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) +{ + return obj->flags & I915_BO_ALLOC_VOLATILE; +} + static inline void i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index eb92243d473b..2142d74a57ea 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -118,7 +118,8 @@ struct drm_i915_gem_object {
unsigned long flags; #define I915_BO_ALLOC_CONTIGUOUS BIT(0) -#define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS) +#define I915_BO_ALLOC_VOLATILE BIT(1) +#define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | I915_BO_ALLOC_VOLATILE)
/* * Is the object to be mapped as read-only to the GPU diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 18f0ce0135c1..d3f0debdb875 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -18,6 +18,9 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
lockdep_assert_held(&obj->mm.lock);
+ if (i915_gem_object_is_volatile(obj)) + obj->mm.madv = I915_MADV_DONTNEED; + /* Make the pages coherent with the GPU (flushing any swapin). */ if (obj->cache_dirty) { obj->write_domain = 0; @@ -159,6 +162,9 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) if (IS_ERR_OR_NULL(pages)) return pages;
+ if (i915_gem_object_is_volatile(obj)) + obj->mm.madv = I915_MADV_WILLNEED; + i915_gem_object_make_unshrinkable(obj);
if (obj->mm.mapping) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index d9cd722b5dbf..0d09da9f7168 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -109,7 +109,12 @@ void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj, obj->flags = flags;
mutex_lock(&mem->obj_lock); - list_add(&obj->mm.region_link, &mem->objects); + + if (obj->flags & I915_BO_ALLOC_VOLATILE) + list_add(&obj->mm.region_link, &mem->purgeable); + else + list_add(&obj->mm.region_link, &mem->objects); + mutex_unlock(&mem->obj_lock); }
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index 4c594f90b776..6ead53455c51 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -115,8 +115,6 @@ static int get_huge_pages(struct drm_i915_gem_object *obj) if (i915_gem_gtt_prepare_pages(obj, st)) goto err;
- obj->mm.madv = I915_MADV_DONTNEED; - GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); __i915_gem_object_set_pages(obj, st, sg_page_sizes);
@@ -137,7 +135,6 @@ static void put_huge_pages(struct drm_i915_gem_object *obj, huge_pages_free_pages(pages);
obj->mm.dirty = false; - obj->mm.madv = I915_MADV_WILLNEED; }
static const struct drm_i915_gem_object_ops huge_page_ops = { @@ -170,6 +167,8 @@ huge_pages_object(struct drm_i915_private *i915, drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &huge_page_ops);
+ obj->flags = I915_BO_ALLOC_VOLATILE; + obj->write_domain = I915_GEM_DOMAIN_CPU; obj->read_domains = I915_GEM_DOMAIN_CPU; obj->cache_level = I915_CACHE_NONE; @@ -229,8 +228,6 @@ static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
i915_sg_trim(st);
- obj->mm.madv = I915_MADV_DONTNEED; - __i915_gem_object_set_pages(obj, st, sg_page_sizes);
return 0; @@ -263,8 +260,6 @@ static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) sg_dma_len(sg) = obj->base.size; sg_dma_address(sg) = page_size;
- obj->mm.madv = I915_MADV_DONTNEED; - __i915_gem_object_set_pages(obj, st, sg->length);
return 0; @@ -283,7 +278,6 @@ static void fake_put_huge_pages(struct drm_i915_gem_object *obj, { fake_free_huge_pages(obj, pages); obj->mm.dirty = false; - obj->mm.madv = I915_MADV_WILLNEED; }
static const struct drm_i915_gem_object_ops fake_ops = { @@ -323,6 +317,8 @@ fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) else i915_gem_object_init(obj, &fake_ops);
+ obj->flags = I915_BO_ALLOC_VOLATILE; + obj->write_domain = I915_GEM_DOMAIN_CPU; obj->read_domains = I915_GEM_DOMAIN_CPU; obj->cache_level = I915_CACHE_NONE; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 31a51ca1ddcb..81850e3a7d2d 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -88,8 +88,6 @@ static int fake_get_pages(struct drm_i915_gem_object *obj) } GEM_BUG_ON(rem);
- obj->mm.madv = I915_MADV_DONTNEED; - __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
return 0; @@ -101,7 +99,6 @@ static void fake_put_pages(struct drm_i915_gem_object *obj, { fake_free_pages(obj, pages); obj->mm.dirty = false; - obj->mm.madv = I915_MADV_WILLNEED; }
static const struct drm_i915_gem_object_ops fake_ops = { @@ -128,6 +125,8 @@ fake_dma_object(struct drm_i915_private *i915, u64 size) drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &fake_ops);
+ obj->flags = I915_BO_ALLOC_VOLATILE; + obj->write_domain = I915_GEM_DOMAIN_CPU; obj->read_domains = I915_GEM_DOMAIN_CPU; obj->cache_level = I915_CACHE_NONE; diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index 70b467d4e811..f94e7eae90e9 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -295,12 +295,68 @@ static int igt_mock_continuous(void *arg) return err; }
+static int igt_mock_volatile(void *arg) +{ + struct intel_memory_region *mem = arg; + struct drm_i915_gem_object *obj; + int err; + + obj = i915_gem_object_create_region(mem, PAGE_SIZE, 0); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + err = i915_gem_object_pin_pages(obj); + if (err) + goto err_put; + + i915_gem_object_unpin_pages(obj); + + err = intel_memory_region_evict(mem, PAGE_SIZE, 0); + if (err != -ENOSPC) { + pr_err("shrink memory region\n"); + goto err_put; + } + + i915_gem_object_put(obj); + + obj = i915_gem_object_create_region(mem, PAGE_SIZE, I915_BO_ALLOC_VOLATILE); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + if (!(obj->flags & I915_BO_ALLOC_VOLATILE)) { + pr_err("missing flags\n"); + goto err_put; + } + + err = i915_gem_object_pin_pages(obj); + if (err) + goto err_put; + + i915_gem_object_unpin_pages(obj); + + err = intel_memory_region_evict(mem, PAGE_SIZE, 0); + if (err) { + pr_err("failed to shrink memory\n"); + goto err_put; + } + + if (i915_gem_object_has_pages(obj)) { + pr_err("object pages not discarded\n"); + err = -EINVAL; + } + +err_put: + i915_gem_object_put(obj); + return err; +} + int intel_memory_region_mock_selftests(void) { static const struct i915_subtest tests[] = { SUBTEST(igt_mock_fill), SUBTEST(igt_mock_shrink), SUBTEST(igt_mock_continuous), + SUBTEST(igt_mock_volatile), }; struct intel_memory_region *mem; struct drm_i915_private *i915;