Implement object migration, needed primarily for dma-buf exports of objects currently residing in LMEM, until we land p2pdma. There are no users yet of this code.
Signed-off-by: Thomas Hellström thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/i915/gem/i915_gem_object.c | 100 ++++++++++++++++++ drivers/gpu/drm/i915/gem/i915_gem_object.h | 10 ++ .../gpu/drm/i915/gem/i915_gem_object_types.h | 7 ++ drivers/gpu/drm/i915/gem/i915_gem_ttm.c | 51 ++++++++- 4 files changed, 164 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 07e8ff9a8aae..1589053ea99e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -475,6 +475,106 @@ bool i915_gem_object_migratable(struct drm_i915_gem_object *obj) return obj->mm.n_placements > 1; }
+bool i915_gem_object_can_migrate_to_region(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr, + unsigned int *placement_index) +{ + unsigned int i; + unsigned int num_allowed = obj->mm.n_placements; + + if (!i915_gem_object_evictable(obj)) + return false; + + if (num_allowed == 0 && mr != obj->mm.region) + return false; + + if (num_allowed == 1 && mr != obj->mm.placements[0]) + return false; + + for (i = 0; i < num_allowed; ++i) { + if (mr == obj->mm.placements[i]) { + if (placement_index) + *placement_index = i; + return true; + } + } + + return false; +} + +/** + * i915_gem_object_migrate_to_region_lazy - Lazily migrate an object + * @obj: The object to migrate. + * @mr: The region to migrate to. + * + * Check that @obj can migrate to @mr, and update all data necessary to + * make that happen on the next get_pages(). We sync and unbind gpu bindings + * and put pages. The word "lazy" means that the actual migration blit + * is not triggered by this function. + * + * Return: Zero on success, negative error code on failure. + */ +int i915_gem_object_migrate_to_region_lazy(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr) +{ + unsigned int index; + int ret; + + if (obj->mm.region == mr) + return 0; + + if (!i915_gem_object_can_migrate_to_region(obj, mr, &index)) + return -EINVAL; + + ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); + if (ret) + return ret; + + ret = __i915_gem_object_put_pages(obj); + if (ret) + return ret; + + /* + * The next get_pages() will pick up the new desired placement + * and migrate. + */ + if (obj->mm.override_region) { + intel_memory_region_put(obj->mm.override_region); + obj->mm.override_region = NULL; + } + + if (index != 0) + obj->mm.override_region = + intel_memory_region_get(obj->mm.placements[index]); + + return 0; +} + +/** + * i915_gem_object_migrate_to_region - Migrate an object + * @obj: The object to migrate. + * @mr: The region to migrate to. + * + * Check that @obj can migrate to @mr, and migrate the object. + * The caller needs to check that the final region was the + * desired one since the object may have ended up elsewhere on + * lack of space in the desired region, and if there are other + * allowed placements. + * + * Return: Zero on success, negative error code on failure. + */ +int i915_gem_object_migrate_to_region(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr) +{ + int ret; + + ret = i915_gem_object_migrate_to_region_lazy(obj, mr); + if (ret) + return ret; + + return ____i915_gem_object_get_pages(obj); +} + /** * i915_gem_object_has_struct_page - Whether the object is page-backed * @obj: The object to query. diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 23e26f6b1db9..dfab5b080c54 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -592,6 +592,16 @@ bool i915_gem_object_migratable(struct drm_i915_gem_object *obj);
bool i915_gem_object_validates_to_lmem(struct drm_i915_gem_object *obj);
+bool i915_gem_object_can_migrate_to_region(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr, + unsigned int *placement_index); + +int i915_gem_object_migrate_to_region_lazy(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr); + +int i915_gem_object_migrate_to_region(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr); + #ifdef CONFIG_MMU_NOTIFIER static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index fb9ccc3f50e7..d645fa6f4c37 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -264,6 +264,13 @@ struct drm_i915_gem_object { */ struct intel_memory_region *region;
+ /** + * Override memory region for this object. Use to + * override the order of the placement list to migrate + * an object to the desired region. + */ + struct intel_memory_region *override_region; + /** * Memory manager node allocated for this object. */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 42e89bf43708..c798470173e2 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -104,19 +104,45 @@ i915_ttm_placement_from_obj(const struct drm_i915_gem_object *obj, { unsigned int i; unsigned int num_allowed = obj->mm.n_placements; + struct intel_memory_region *requested_mr; + unsigned int swap_index = 0;
placement->num_placement = 1; - i915_ttm_place_from_region(num_allowed ? obj->mm.placements[0] : - obj->mm.region, requested);
- /* Cache this on object? */ + /* + * We migrate by setting the override region to something sensible. + * Warn if the override region is not in the allowed regions list + */ + if (obj->mm.override_region && !GEM_WARN_ON(!num_allowed)) { + requested_mr = obj->mm.override_region; + swap_index = num_allowed; + } else if (num_allowed) { + requested_mr = obj->mm.placements[0]; + } else { + requested_mr = obj->mm.region; + } + i915_ttm_place_from_region(requested_mr, requested); + + /* In the future we might want to cache the busy list on the object? */ placement->num_busy_placement = num_allowed; - for (i = 0; i < placement->num_busy_placement; ++i) + for (i = 0; i < placement->num_busy_placement; ++i) { i915_ttm_place_from_region(obj->mm.placements[i], busy + i); + if (requested_mr == obj->mm.placements[i]) + swap_index = i; + } + + /* Override region was not in placement list. */ + if (num_allowed && GEM_WARN_ON(swap_index == num_allowed)) { + swap_index = 0; + i915_ttm_place_from_region(obj->mm.placements[0], requested); + }
if (num_allowed == 0) { *busy = *requested; placement->num_busy_placement = 1; + } else if (swap_index != 0) { + /* Put the override placement highest in the busy list */ + swap(busy[0], busy[swap_index]); }
placement->placement = requested; @@ -246,6 +272,23 @@ static void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj) struct drm_i915_private *i915 = to_i915(obj->base.dev); struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); unsigned int cache_level; + unsigned int i; + + /* + * If object was moved to an allowable region, update the object + * region to consider it migrated. Note that if it's currently not + * in an allowable region, it's evicted and we don't update the + * object region. + */ + for (i = 0; i < obj->mm.n_placements; ++i) { + struct intel_memory_region *mr = obj->mm.placements[i]; + + if (intel_region_to_ttm_type(mr) == bo->mem.mem_type && + mr != obj->mm.region) { + intel_memory_region_put(obj->mm.region); + obj->mm.region = intel_memory_region_get(mr); + } + }
obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM);