- Make drm_buddy_alloc a single function to handle range allocation and non-range allocation demands
- Implemented a new function alloc_range() which allocates the requested power-of-two block comply with range limitations
- Moved order computation and memory alignment logic from i915 driver to drm buddy
v2: merged below changes to keep the build unbroken - drm_buddy_alloc_range() becomes obsolete and may be removed - enable ttm range allocation (fpfn / lpfn) support in i915 driver - apply enhanced drm_buddy_alloc() function to i915 driver
v3(Matthew Auld): - Fix alignment issues and remove unnecessary list_empty check - add more validation checks for input arguments - make alloc_range() block allocations as bottom-up - optimize order computation logic - replace uint64_t with u64, which is preferred in the kernel
v4(Matthew Auld): - keep drm_buddy_alloc_range() function implementation for generic actual range allocations - keep alloc_range() implementation for end bias allocations
v5(Matthew Auld): - modify drm_buddy_alloc() passing argument place->lpfn to lpfn as place->lpfn will currently always be zero for i915
v6(Matthew Auld): - fixup potential uaf - If we are unlucky and can't allocate enough memory when splitting blocks, where we temporarily end up with the given block and its buddy on the respective free list, then we need to ensure we delete both blocks, and no just the buddy, before potentially freeing them
- fix warnings reported by kernel test robot lkp@intel.com
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com --- drivers/gpu/drm/drm_buddy.c | 326 +++++++++++++----- drivers/gpu/drm/i915/i915_ttm_buddy_manager.c | 67 ++-- drivers/gpu/drm/i915/i915_ttm_buddy_manager.h | 2 + include/drm/drm_buddy.h | 22 +- 4 files changed, 293 insertions(+), 124 deletions(-)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index d60878bc9c20..954e31962c74 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -282,23 +282,99 @@ void drm_buddy_free_list(struct drm_buddy *mm, struct list_head *objects) } EXPORT_SYMBOL(drm_buddy_free_list);
-/** - * drm_buddy_alloc_blocks - allocate power-of-two blocks - * - * @mm: DRM buddy manager to allocate from - * @order: size of the allocation - * - * The order value here translates to: - * - * 0 = 2^0 * mm->chunk_size - * 1 = 2^1 * mm->chunk_size - * 2 = 2^2 * mm->chunk_size - * - * Returns: - * allocated ptr to the &drm_buddy_block on success - */ -struct drm_buddy_block * -drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order) +static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) +{ + return s1 <= e2 && e1 >= s2; +} + +static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) +{ + return s1 <= s2 && e1 >= e2; +} + +static struct drm_buddy_block * +alloc_range_bias(struct drm_buddy *mm, + u64 start, u64 end, + unsigned int order) +{ + struct drm_buddy_block *block; + struct drm_buddy_block *buddy; + LIST_HEAD(dfs); + int err; + int i; + + end = end - 1; + + for (i = 0; i < mm->n_roots; ++i) + list_add_tail(&mm->roots[i]->tmp_link, &dfs); + + do { + u64 block_start; + u64 block_end; + + block = list_first_entry_or_null(&dfs, + struct drm_buddy_block, + tmp_link); + if (!block) + break; + + list_del(&block->tmp_link); + + if (drm_buddy_block_order(block) < order) + continue; + + block_start = drm_buddy_block_offset(block); + block_end = block_start + drm_buddy_block_size(mm, block) - 1; + + if (!overlaps(start, end, block_start, block_end)) + continue; + + if (drm_buddy_block_is_allocated(block)) + continue; + + if (contains(start, end, block_start, block_end) && + order == drm_buddy_block_order(block)) { + /* + * Find the free block within the range. + */ + if (drm_buddy_block_is_free(block)) + return block; + + continue; + } + + if (!drm_buddy_block_is_split(block)) { + err = split_block(mm, block); + if (unlikely(err)) + goto err_undo; + } + + list_add(&block->right->tmp_link, &dfs); + list_add(&block->left->tmp_link, &dfs); + } while (1); + + return ERR_PTR(-ENOSPC); + +err_undo: + /* + * We really don't want to leave around a bunch of split blocks, since + * bigger is better, so make sure we merge everything back before we + * free the allocated blocks. + */ + buddy = get_buddy(block); + if (buddy && + (drm_buddy_block_is_free(block) && + drm_buddy_block_is_free(buddy))) { + list_del(&block->link); + __drm_buddy_free(mm, block); + } + return ERR_PTR(err); +} + +static struct drm_buddy_block * +alloc_from_freelist(struct drm_buddy *mm, + unsigned int order, + unsigned long flags) { struct drm_buddy_block *block = NULL; unsigned int i; @@ -320,78 +396,30 @@ drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order) while (i != order) { err = split_block(mm, block); if (unlikely(err)) - goto out_free; + goto err_undo;
- /* Go low */ - block = block->left; + block = block->right; i--; } - - mark_allocated(block); - mm->avail -= drm_buddy_block_size(mm, block); - kmemleak_update_trace(block); return block;
-out_free: - if (i != order) +err_undo: + if (i != order) { + list_del(&block->link); __drm_buddy_free(mm, block); + } return ERR_PTR(err); } -EXPORT_SYMBOL(drm_buddy_alloc_blocks); - -static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) -{ - return s1 <= e2 && e1 >= s2; -}
-static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) -{ - return s1 <= s2 && e1 >= e2; -} - -/** - * drm_buddy_alloc_range - allocate range - * - * @mm: DRM buddy manager to allocate from - * @blocks: output list head to add allocated blocks - * @start: start of the allowed range for this block - * @size: size of the allocation - * - * Intended for pre-allocating portions of the address space, for example to - * reserve a block for the initial framebuffer or similar, hence the expectation - * here is that drm_buddy_alloc_blocks() is still the main vehicle for - * allocations, so if that's not the case then the drm_mm range allocator is - * probably a much better fit, and so you should probably go use that instead. - * - * Note that it's safe to chain together multiple alloc_ranges - * with the same blocks list - * - * Returns: - * 0 on success, error code on failure. - */ -int drm_buddy_alloc_range(struct drm_buddy *mm, - struct list_head *blocks, - u64 start, u64 size) +static int __alloc_range(struct drm_buddy *mm, + struct list_head *dfs, + u64 start, u64 size, + struct list_head *blocks) { struct drm_buddy_block *block; struct drm_buddy_block *buddy; - LIST_HEAD(allocated); - LIST_HEAD(dfs); u64 end; int err; - int i; - - if (size < mm->chunk_size) - return -EINVAL; - - if (!IS_ALIGNED(size | start, mm->chunk_size)) - return -EINVAL; - - if (range_overflows(start, size, mm->size)) - return -EINVAL; - - for (i = 0; i < mm->n_roots; ++i) - list_add_tail(&mm->roots[i]->tmp_link, &dfs);
end = start + size - 1;
@@ -399,7 +427,7 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, u64 block_start; u64 block_end;
- block = list_first_entry_or_null(&dfs, + block = list_first_entry_or_null(dfs, struct drm_buddy_block, tmp_link); if (!block) @@ -426,7 +454,7 @@ int drm_buddy_alloc_range(struct drm_buddy *mm,
mark_allocated(block); mm->avail -= drm_buddy_block_size(mm, block); - list_add_tail(&block->link, &allocated); + list_add_tail(&block->link, blocks); continue; }
@@ -436,11 +464,10 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, goto err_undo; }
- list_add(&block->right->tmp_link, &dfs); - list_add(&block->left->tmp_link, &dfs); + list_add(&block->right->tmp_link, dfs); + list_add(&block->left->tmp_link, dfs); } while (1);
- list_splice_tail(&allocated, blocks); return 0;
err_undo: @@ -452,14 +479,149 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, buddy = get_buddy(block); if (buddy && (drm_buddy_block_is_free(block) && - drm_buddy_block_is_free(buddy))) + drm_buddy_block_is_free(buddy))) { + list_del(&block->link); __drm_buddy_free(mm, block); + } + +err_free: + drm_buddy_free_list(mm, blocks); + return err; +} + +/** + * __drm_buddy_alloc_range - actual range allocation + * + * @mm: DRM buddy manager to allocate from + * @start: start of the allowed range for this block + * @size: size of the allocation + * @blocks: output list head to add allocated blocks + * + * Intended for pre-allocating portions of the address space, for example to + * reserve a block for the initial framebuffer or similar + * + * Note that it's safe to chain together multiple alloc_ranges + * with the same blocks list + * + * Returns: + * 0 on success, error code on failure. + */ +static int __drm_buddy_alloc_range(struct drm_buddy *mm, + u64 start, + u64 size, + struct list_head *blocks) +{ + LIST_HEAD(dfs); + int i; + + for (i = 0; i < mm->n_roots; ++i) + list_add_tail(&mm->roots[i]->tmp_link, &dfs); + + return __alloc_range(mm, &dfs, start, size, blocks); +} + +/** + * drm_buddy_alloc_blocks - allocate power-of-two blocks + * + * @mm: DRM buddy manager to allocate from + * @start: start of the allowed range for this block + * @end: end of the allowed range for this block + * @size: size of the allocation + * @min_page_size: alignment of the allocation + * @blocks: output list head to add allocated blocks + * @flags: DRM_BUDDY_*_ALLOCATION flags + * + * alloc_range_bias() called on range limitations, which traverses + * the tree and returns the desired block. + * + * alloc_from_freelist() called when *no* range restrictions + * are enforced, which picks the block from the freelist. + * + * blocks are allocated in order, the order value here translates to: + * + * 0 = 2^0 * mm->chunk_size + * 1 = 2^1 * mm->chunk_size + * 2 = 2^2 * mm->chunk_size + * + * Returns: + * 0 on success, error code on failure. + */ +int drm_buddy_alloc_blocks(struct drm_buddy *mm, + u64 start, u64 end, u64 size, + u64 min_page_size, + struct list_head *blocks, + unsigned long flags) +{ + struct drm_buddy_block *block = NULL; + unsigned int min_order, order; + unsigned long pages; + LIST_HEAD(allocated); + int err; + + if (size < mm->chunk_size) + return -EINVAL; + + if (min_page_size < mm->chunk_size) + return -EINVAL; + + if (!is_power_of_2(min_page_size)) + return -EINVAL; + + if (!IS_ALIGNED(start | end | size, mm->chunk_size)) + return -EINVAL; + + if (check_range_overflow(start, end, size, mm->size)) + return -EINVAL; + + /* Actual range allocation */ + if (start + size == end) + return __drm_buddy_alloc_range(mm, start, size, blocks); + + pages = size >> ilog2(mm->chunk_size); + order = fls(pages) - 1; + min_order = ilog2(min_page_size) - ilog2(mm->chunk_size); + + do { + order = min(order, (unsigned int)fls(pages) - 1); + BUG_ON(order > mm->max_order); + BUG_ON(order < min_order); + + do { + if (flags & DRM_BUDDY_RANGE_ALLOCATION) + /* Allocate traversing within the range */ + block = alloc_range_bias(mm, start, end, order); + else + /* Allocate from freelist */ + block = alloc_from_freelist(mm, order, flags); + + if (!IS_ERR(block)) + break; + + if (order-- == min_order) { + err = -ENOSPC; + goto err_free; + } + } while (1); + + mark_allocated(block); + mm->avail -= drm_buddy_block_size(mm, block); + kmemleak_update_trace(block); + list_add_tail(&block->link, &allocated); + + pages -= BIT(order); + + if (!pages) + break; + } while (1); + + list_splice_tail(&allocated, blocks); + return 0;
err_free: drm_buddy_free_list(mm, &allocated); return err; } -EXPORT_SYMBOL(drm_buddy_alloc_range); +EXPORT_SYMBOL(drm_buddy_alloc_blocks);
/** * drm_buddy_block_print - print block information diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 6ba314f9836a..1411f4cf1f21 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -36,13 +36,14 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct i915_ttm_buddy_resource *bman_res; struct drm_buddy *mm = &bman->mm; - unsigned long n_pages; - unsigned int min_order; + unsigned long n_pages, lpfn; u64 min_page_size; u64 size; int err;
- GEM_BUG_ON(place->fpfn || place->lpfn); + lpfn = place->lpfn; + if (!lpfn) + lpfn = man->size;
bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL); if (!bman_res) @@ -52,6 +53,9 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, INIT_LIST_HEAD(&bman_res->blocks); bman_res->mm = mm;
+ if (place->fpfn || lpfn != man->size) + bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION; + GEM_BUG_ON(!bman_res->base.num_pages); size = bman_res->base.num_pages << PAGE_SHIFT;
@@ -60,10 +64,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, min_page_size = bo->page_alignment << PAGE_SHIFT;
GEM_BUG_ON(min_page_size < mm->chunk_size); - min_order = ilog2(min_page_size) - ilog2(mm->chunk_size); + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + unsigned long pages; + size = roundup_pow_of_two(size); - min_order = ilog2(size) - ilog2(mm->chunk_size); + min_page_size = size; + + pages = size >> ilog2(mm->chunk_size); + if (pages > lpfn) + lpfn = pages; }
if (size > mm->size) { @@ -73,34 +83,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
n_pages = size >> ilog2(mm->chunk_size);
- do { - struct drm_buddy_block *block; - unsigned int order; - - order = fls(n_pages) - 1; - GEM_BUG_ON(order > mm->max_order); - GEM_BUG_ON(order < min_order); - - do { - mutex_lock(&bman->lock); - block = drm_buddy_alloc_blocks(mm, order); - mutex_unlock(&bman->lock); - if (!IS_ERR(block)) - break; - - if (order-- == min_order) { - err = -ENOSPC; - goto err_free_blocks; - } - } while (1); - - n_pages -= BIT(order); - - list_add_tail(&block->link, &bman_res->blocks); - - if (!n_pages) - break; - } while (1); + mutex_lock(&bman->lock); + err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, + (u64)lpfn << PAGE_SHIFT, + (u64)n_pages << PAGE_SHIFT, + min_page_size, + &bman_res->blocks, + bman_res->flags); + mutex_unlock(&bman->lock); + if (unlikely(err)) + goto err_free_blocks;
*res = &bman_res->base; return 0; @@ -266,10 +258,17 @@ int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct drm_buddy *mm = &bman->mm; + unsigned long flags = 0; int ret;
+ flags |= DRM_BUDDY_RANGE_ALLOCATION; + mutex_lock(&bman->lock); - ret = drm_buddy_alloc_range(mm, &bman->reserved, start, size); + ret = drm_buddy_alloc_blocks(mm, start, + start + size, + size, mm->chunk_size, + &bman->reserved, + flags); mutex_unlock(&bman->lock);
return ret; diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h index 312077941411..72c90b432e87 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h @@ -20,6 +20,7 @@ struct drm_buddy; * * @base: struct ttm_resource base class we extend * @blocks: the list of struct i915_buddy_block for this resource/allocation + * @flags: DRM_BUDDY_*_ALLOCATION flags * @mm: the struct i915_buddy_mm for this resource * * Extends the struct ttm_resource to manage an address space allocation with @@ -28,6 +29,7 @@ struct drm_buddy; struct i915_ttm_buddy_resource { struct ttm_resource base; struct list_head blocks; + unsigned long flags; struct drm_buddy *mm; };
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index f524db152413..865664b90a8a 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -13,15 +13,22 @@
#include <drm/drm_print.h>
-#define range_overflows(start, size, max) ({ \ +#define check_range_overflow(start, end, size, max) ({ \ typeof(start) start__ = (start); \ + typeof(end) end__ = (end);\ typeof(size) size__ = (size); \ typeof(max) max__ = (max); \ (void)(&start__ == &size__); \ (void)(&start__ == &max__); \ - start__ >= max__ || size__ > max__ - start__; \ + (void)(&start__ == &end__); \ + (void)(&end__ == &size__); \ + (void)(&end__ == &max__); \ + start__ >= max__ || end__ > max__ || \ + size__ > end__ - start__; \ })
+#define DRM_BUDDY_RANGE_ALLOCATION (1 << 0) + struct drm_buddy_block { #define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) #define DRM_BUDDY_HEADER_STATE GENMASK_ULL(11, 10) @@ -131,12 +138,11 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size);
void drm_buddy_fini(struct drm_buddy *mm);
-struct drm_buddy_block * -drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order); - -int drm_buddy_alloc_range(struct drm_buddy *mm, - struct list_head *blocks, - u64 start, u64 size); +int drm_buddy_alloc_blocks(struct drm_buddy *mm, + u64 start, u64 end, u64 size, + u64 min_page_size, + struct list_head *blocks, + unsigned long flags);
void drm_buddy_free_block(struct drm_buddy *mm, struct drm_buddy_block *block);
base-commit: e3a3896bf9a189f3992db9fd8622893cf90652cb
Implemented a function which walk through the order list, compares the offset and returns the maximum offset block, this method is unpredictable in obtaining the high range address blocks which depends on allocation and deallocation. for instance, if driver requests address at a low specific range, allocator traverses from the root block and splits the larger blocks until it reaches the specific block and in the process of splitting, lower orders in the freelist are occupied with low range address blocks and for the subsequent TOPDOWN memory request we may return the low range blocks.To overcome this issue, we may go with the below approach.
The other approach, sorting each order list entries in ascending order and compares the last entry of each order list in the freelist and return the max block. This creates sorting overhead on every drm_buddy_free() request and split up of larger blocks for a single page request.
v2: - Fix alignment issues(Matthew Auld) - Remove unnecessary list_empty check(Matthew Auld) - merged the below patch to see the feature in action - add top-down alloc support to i915 driver
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com --- drivers/gpu/drm/drm_buddy.c | 36 ++++++++++++++++--- drivers/gpu/drm/i915/i915_ttm_buddy_manager.c | 3 ++ include/drm/drm_buddy.h | 1 + 3 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index 954e31962c74..6aa5c1ce25bf 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -371,6 +371,26 @@ alloc_range_bias(struct drm_buddy *mm, return ERR_PTR(err); }
+static struct drm_buddy_block * +get_maxblock(struct list_head *head) +{ + struct drm_buddy_block *max_block = NULL, *node; + + max_block = list_first_entry_or_null(head, + struct drm_buddy_block, + link); + if (!max_block) + return NULL; + + list_for_each_entry(node, head, link) { + if (drm_buddy_block_offset(node) > + drm_buddy_block_offset(max_block)) + max_block = node; + } + + return max_block; +} + static struct drm_buddy_block * alloc_from_freelist(struct drm_buddy *mm, unsigned int order, @@ -381,11 +401,17 @@ alloc_from_freelist(struct drm_buddy *mm, int err;
for (i = order; i <= mm->max_order; ++i) { - block = list_first_entry_or_null(&mm->free_list[i], - struct drm_buddy_block, - link); - if (block) - break; + if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) { + block = get_maxblock(&mm->free_list[i]); + if (block) + break; + } else { + block = list_first_entry_or_null(&mm->free_list[i], + struct drm_buddy_block, + link); + if (block) + break; + } }
if (!block) diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 1411f4cf1f21..3662434b64bb 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -53,6 +53,9 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, INIT_LIST_HEAD(&bman_res->blocks); bman_res->mm = mm;
+ if (place->flags & TTM_PL_FLAG_TOPDOWN) + bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + if (place->fpfn || lpfn != man->size) bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index 865664b90a8a..424fc443115e 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -28,6 +28,7 @@ })
#define DRM_BUDDY_RANGE_ALLOCATION (1 << 0) +#define DRM_BUDDY_TOPDOWN_ALLOCATION (1 << 1)
struct drm_buddy_block { #define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12)
On 19/01/2022 11:37, Arunpravin wrote:
Implemented a function which walk through the order list, compares the offset and returns the maximum offset block, this method is unpredictable in obtaining the high range address blocks which depends on allocation and deallocation. for instance, if driver requests address at a low specific range, allocator traverses from the root block and splits the larger blocks until it reaches the specific block and in the process of splitting, lower orders in the freelist are occupied with low range address blocks and for the subsequent TOPDOWN memory request we may return the low range blocks.To overcome this issue, we may go with the below approach.
The other approach, sorting each order list entries in ascending order and compares the last entry of each order list in the freelist and return the max block. This creates sorting overhead on every drm_buddy_free() request and split up of larger blocks for a single page request.
v2:
- Fix alignment issues(Matthew Auld)
- Remove unnecessary list_empty check(Matthew Auld)
- merged the below patch to see the feature in action
- add top-down alloc support to i915 driver
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com
drivers/gpu/drm/drm_buddy.c | 36 ++++++++++++++++--- drivers/gpu/drm/i915/i915_ttm_buddy_manager.c | 3 ++ include/drm/drm_buddy.h | 1 + 3 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index 954e31962c74..6aa5c1ce25bf 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -371,6 +371,26 @@ alloc_range_bias(struct drm_buddy *mm, return ERR_PTR(err); }
+static struct drm_buddy_block * +get_maxblock(struct list_head *head) +{
- struct drm_buddy_block *max_block = NULL, *node;
- max_block = list_first_entry_or_null(head,
struct drm_buddy_block,link);- if (!max_block)
return NULL;- list_for_each_entry(node, head, link) {
if (drm_buddy_block_offset(node) >drm_buddy_block_offset(max_block))max_block = node;- }
If we feed in the knowledge of the visible_size(or perhaps implement that generically as "zones"), I think this can be done more efficiently. It could also be useful to track directly in the allocator how much of the visible_size is still available, rather than having to do that in the upper levels by scanning the entire list. But hopefully in practice this should be good enough for our needs, Reviewed-by: Matthew Auld matthew.auld@intel.com
- return max_block;
+}
- static struct drm_buddy_block * alloc_from_freelist(struct drm_buddy *mm, unsigned int order,
@@ -381,11 +401,17 @@ alloc_from_freelist(struct drm_buddy *mm, int err;
for (i = order; i <= mm->max_order; ++i) {
block = list_first_entry_or_null(&mm->free_list[i],struct drm_buddy_block,link);if (block)break;
if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) {block = get_maxblock(&mm->free_list[i]);if (block)break;} else {block = list_first_entry_or_null(&mm->free_list[i],struct drm_buddy_block,link);if (block)break;}}
if (!block)
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 1411f4cf1f21..3662434b64bb 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -53,6 +53,9 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, INIT_LIST_HEAD(&bman_res->blocks); bman_res->mm = mm;
- if (place->flags & TTM_PL_FLAG_TOPDOWN)
bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;- if (place->fpfn || lpfn != man->size) bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index 865664b90a8a..424fc443115e 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -28,6 +28,7 @@ })
#define DRM_BUDDY_RANGE_ALLOCATION (1 << 0) +#define DRM_BUDDY_TOPDOWN_ALLOCATION (1 << 1)
struct drm_buddy_block { #define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12)
On contiguous allocation, we round up the size to the *next* power of 2, implement a function to free the unused pages after the newly allocate block.
v2(Matthew Auld): - replace function name 'drm_buddy_free_unused_pages' with drm_buddy_block_trim - replace input argument name 'actual_size' with 'new_size' - add more validation checks for input arguments - add overlaps check to avoid needless searching and splitting - merged the below patch to see the feature in action - add free unused pages support to i915 driver - lock drm_buddy_block_trim() function as it calls mark_free/mark_split are all globally visible
v3(Matthew Auld): - remove trim method error handling as we address the failure case at drm_buddy_block_trim() function
v4: - in case of trim, at __alloc_range() split_block failure path marks the block as free and removes it from the original list, potentially also freeing it, to overcome this problem, we turn the drm_buddy_block_trim() input node into a temporary node to prevent recursively freeing itself, but still retain the un-splitting/freeing of the other nodes(Matthew Auld)
- modify the drm_buddy_block_trim() function return type
v5(Matthew Auld): - revert drm_buddy_block_trim() function return type changes in v4 - modify drm_buddy_block_trim() passing argument n_pages to original_size as n_pages has already been rounded up to the next power-of-two and passing n_pages results noop
v6: - fix warnings reported by kernel test robot lkp@intel.com
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com --- drivers/gpu/drm/drm_buddy.c | 65 +++++++++++++++++++ drivers/gpu/drm/i915/i915_ttm_buddy_manager.c | 10 +++ include/drm/drm_buddy.h | 4 ++ 3 files changed, 79 insertions(+)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index 6aa5c1ce25bf..c5902a81b8c5 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -546,6 +546,71 @@ static int __drm_buddy_alloc_range(struct drm_buddy *mm, return __alloc_range(mm, &dfs, start, size, blocks); }
+/** + * drm_buddy_block_trim - free unused pages + * + * @mm: DRM buddy manager + * @new_size: original size requested + * @blocks: output list head to add allocated blocks + * + * For contiguous allocation, we round up the size to the nearest + * power of two value, drivers consume *actual* size, so remaining + * portions are unused and it can be freed. + * + * Returns: + * 0 on success, error code on failure. + */ +int drm_buddy_block_trim(struct drm_buddy *mm, + u64 new_size, + struct list_head *blocks) +{ + struct drm_buddy_block *parent; + struct drm_buddy_block *block; + LIST_HEAD(dfs); + u64 new_start; + int err; + + if (!list_is_singular(blocks)) + return -EINVAL; + + block = list_first_entry(blocks, + struct drm_buddy_block, + link); + + if (!drm_buddy_block_is_allocated(block)) + return -EINVAL; + + if (new_size > drm_buddy_block_size(mm, block)) + return -EINVAL; + + if (!new_size && !IS_ALIGNED(new_size, mm->chunk_size)) + return -EINVAL; + + if (new_size == drm_buddy_block_size(mm, block)) + return 0; + + list_del(&block->link); + mark_free(mm, block); + mm->avail += drm_buddy_block_size(mm, block); + + /* Prevent recursively freeing this node */ + parent = block->parent; + block->parent = NULL; + + new_start = drm_buddy_block_offset(block); + list_add(&block->tmp_link, &dfs); + err = __alloc_range(mm, &dfs, new_start, new_size, blocks); + if (err) { + mark_allocated(block); + mm->avail -= drm_buddy_block_size(mm, block); + list_add(&block->link, blocks); + } + + block->parent = parent; + return err; +} +EXPORT_SYMBOL(drm_buddy_block_trim); + /** * drm_buddy_alloc_blocks - allocate power-of-two blocks * diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 3662434b64bb..53eb100688a6 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -97,6 +97,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, if (unlikely(err)) goto err_free_blocks;
+ if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + u64 original_size = (u64)bman_res->base.num_pages << PAGE_SHIFT; + + mutex_lock(&bman->lock); + drm_buddy_block_trim(mm, + original_size, + &bman_res->blocks); + mutex_unlock(&bman->lock); + } + *res = &bman_res->base; return 0;
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index 424fc443115e..17ca928fce8e 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -145,6 +145,10 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm, struct list_head *blocks, unsigned long flags);
+int drm_buddy_block_trim(struct drm_buddy *mm, + u64 new_size, + struct list_head *blocks); + void drm_buddy_free_block(struct drm_buddy *mm, struct drm_buddy_block *block);
void drm_buddy_free_list(struct drm_buddy *mm, struct list_head *objects);
On 19/01/2022 11:37, Arunpravin wrote:
On contiguous allocation, we round up the size to the *next* power of 2, implement a function to free the unused pages after the newly allocate block.
v2(Matthew Auld):
- replace function name 'drm_buddy_free_unused_pages' with drm_buddy_block_trim
- replace input argument name 'actual_size' with 'new_size'
- add more validation checks for input arguments
- add overlaps check to avoid needless searching and splitting
- merged the below patch to see the feature in action
- add free unused pages support to i915 driver
- lock drm_buddy_block_trim() function as it calls mark_free/mark_split are all globally visible
v3(Matthew Auld):
- remove trim method error handling as we address the failure case at drm_buddy_block_trim() function
v4:
in case of trim, at __alloc_range() split_block failure path marks the block as free and removes it from the original list, potentially also freeing it, to overcome this problem, we turn the drm_buddy_block_trim() input node into a temporary node to prevent recursively freeing itself, but still retain the un-splitting/freeing of the other nodes(Matthew Auld)
modify the drm_buddy_block_trim() function return type
v5(Matthew Auld):
- revert drm_buddy_block_trim() function return type changes in v4
- modify drm_buddy_block_trim() passing argument n_pages to original_size as n_pages has already been rounded up to the next power-of-two and passing n_pages results noop
v6:
- fix warnings reported by kernel test robot lkp@intel.com
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com
drivers/gpu/drm/drm_buddy.c | 65 +++++++++++++++++++ drivers/gpu/drm/i915/i915_ttm_buddy_manager.c | 10 +++ include/drm/drm_buddy.h | 4 ++ 3 files changed, 79 insertions(+)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index 6aa5c1ce25bf..c5902a81b8c5 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -546,6 +546,71 @@ static int __drm_buddy_alloc_range(struct drm_buddy *mm, return __alloc_range(mm, &dfs, start, size, blocks); }
+/**
- drm_buddy_block_trim - free unused pages
- @mm: DRM buddy manager
- @new_size: original size requested
- @blocks: output list head to add allocated blocks
@blocks: Input and output list of allocated blocks. MUST contain single block as input to be trimmed. On success will contain the newly allocated blocks making up the @new_size. Blocks always appear in ascending order.
?
- For contiguous allocation, we round up the size to the nearest
- power of two value, drivers consume *actual* size, so remaining
- portions are unused and it can be freed.
so remaining portions are unused and can be optionally freed with this function.
?
- Returns:
- 0 on success, error code on failure.
- */
+int drm_buddy_block_trim(struct drm_buddy *mm,
u64 new_size,struct list_head *blocks)+{
- struct drm_buddy_block *parent;
- struct drm_buddy_block *block;
- LIST_HEAD(dfs);
- u64 new_start;
- int err;
- if (!list_is_singular(blocks))
return -EINVAL;- block = list_first_entry(blocks,
struct drm_buddy_block,link);- if (!drm_buddy_block_is_allocated(block))
Maybe:
if (WARN_ON(!drm_buddy_block_is_allocated()))
AFAIK it should be normally impossible to be handed such non-allocated block, and so should be treated as a serious programmer error.
?
return -EINVAL;- if (new_size > drm_buddy_block_size(mm, block))
return -EINVAL;- if (!new_size && !IS_ALIGNED(new_size, mm->chunk_size))
return -EINVAL;
I assume that's a typo:
if (!new_size || ...)
Otherwise I think looks good. Some unit tests for this would be nice, but not a blocker. And this does at least pass the igt_mock_contiguous selftest, and I didn't see anything nasty when running on DG1, which does make use of TTM_PL_FLAG_CONTIGUOUS, Reviewed-by: Matthew Auld matthew.auld@intel.com
- if (new_size == drm_buddy_block_size(mm, block))
return 0;- list_del(&block->link);
- mark_free(mm, block);
- mm->avail += drm_buddy_block_size(mm, block);
- /* Prevent recursively freeing this node */
- parent = block->parent;
- block->parent = NULL;
- new_start = drm_buddy_block_offset(block);
- list_add(&block->tmp_link, &dfs);
- err = __alloc_range(mm, &dfs, new_start, new_size, blocks);
- if (err) {
mark_allocated(block);mm->avail -= drm_buddy_block_size(mm, block);list_add(&block->link, blocks);- }
- block->parent = parent;
- return err;
+} +EXPORT_SYMBOL(drm_buddy_block_trim);
- /**
- drm_buddy_alloc_blocks - allocate power-of-two blocks
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 3662434b64bb..53eb100688a6 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -97,6 +97,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, if (unlikely(err)) goto err_free_blocks;
- if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
u64 original_size = (u64)bman_res->base.num_pages << PAGE_SHIFT;mutex_lock(&bman->lock);drm_buddy_block_trim(mm,original_size,&bman_res->blocks);mutex_unlock(&bman->lock);- }
- *res = &bman_res->base; return 0;
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index 424fc443115e..17ca928fce8e 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -145,6 +145,10 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm, struct list_head *blocks, unsigned long flags);
+int drm_buddy_block_trim(struct drm_buddy *mm,
u64 new_size,struct list_head *blocks);void drm_buddy_free_block(struct drm_buddy *mm, struct drm_buddy_block *block);
void drm_buddy_free_list(struct drm_buddy *mm, struct list_head *objects);
Move shared vram inline functions and structs into a header file
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h new file mode 100644 index 000000000000..59983464cce5 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: MIT + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_VRAM_MGR_H__ +#define __AMDGPU_VRAM_MGR_H__ + +#include <drm/drm_buddy.h> + +struct amdgpu_vram_mgr_node { + struct ttm_resource base; + struct list_head blocks; + unsigned long flags; +}; + +static inline u64 amdgpu_node_start(struct drm_buddy_block *block) +{ + return drm_buddy_block_offset(block); +} + +static inline u64 amdgpu_node_size(struct drm_buddy_block *block) +{ + return PAGE_SIZE << drm_buddy_block_order(block); +} + +static inline struct amdgpu_vram_mgr_node * +to_amdgpu_vram_mgr_node(struct ttm_resource *res) +{ + return container_of(res, struct amdgpu_vram_mgr_node, base); +} + +#endif
- Remove drm_mm references and replace with drm buddy functionalities - Add res cursor support for drm buddy
v2(Matthew Auld): - replace spinlock with mutex as we call kmem_cache_zalloc (..., GFP_KERNEL) in drm_buddy_alloc() function
- lock drm_buddy_block_trim() function as it calls mark_free/mark_split are all globally visible
v3(Matthew Auld): - remove trim method error handling as we address the failure case at drm_buddy_block_trim() function
v4: - fix warnings reported by kernel test robot lkp@intel.com
v5: - fix merge conflict issue
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com --- drivers/gpu/drm/Kconfig | 1 + .../gpu/drm/amd/amdgpu/amdgpu_res_cursor.h | 97 +++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 259 ++++++++++-------- 4 files changed, 231 insertions(+), 133 deletions(-)
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index dfdd3ec5f793..eb5a57ae3c5c 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -279,6 +279,7 @@ config DRM_AMDGPU select HWMON select BACKLIGHT_CLASS_DEVICE select INTERVAL_TREE + select DRM_BUDDY help Choose this option if you have a recent AMD Radeon graphics card.
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h index acfa207cf970..da12b4ff2e45 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h @@ -30,12 +30,15 @@ #include <drm/ttm/ttm_resource.h> #include <drm/ttm/ttm_range_manager.h>
+#include "amdgpu_vram_mgr.h" + /* state back for walking over vram_mgr and gtt_mgr allocations */ struct amdgpu_res_cursor { uint64_t start; uint64_t size; uint64_t remaining; - struct drm_mm_node *node; + void *node; + uint32_t mem_type; };
/** @@ -52,27 +55,63 @@ static inline void amdgpu_res_first(struct ttm_resource *res, uint64_t start, uint64_t size, struct amdgpu_res_cursor *cur) { + struct drm_buddy_block *block; + struct list_head *head, *next; struct drm_mm_node *node;
- if (!res || res->mem_type == TTM_PL_SYSTEM) { - cur->start = start; - cur->size = size; - cur->remaining = size; - cur->node = NULL; - WARN_ON(res && start + size > res->num_pages << PAGE_SHIFT); - return; - } + if (!res) + goto err_out;
BUG_ON(start + size > res->num_pages << PAGE_SHIFT);
- node = to_ttm_range_mgr_node(res)->mm_nodes; - while (start >= node->size << PAGE_SHIFT) - start -= node++->size << PAGE_SHIFT; + cur->mem_type = res->mem_type; + + switch (cur->mem_type) { + case TTM_PL_VRAM: + head = &to_amdgpu_vram_mgr_node(res)->blocks; + + block = list_first_entry_or_null(head, + struct drm_buddy_block, + link); + if (!block) + goto err_out; + + while (start >= amdgpu_node_size(block)) { + start -= amdgpu_node_size(block); + + next = block->link.next; + if (next != head) + block = list_entry(next, struct drm_buddy_block, link); + } + + cur->start = amdgpu_node_start(block) + start; + cur->size = min(amdgpu_node_size(block) - start, size); + cur->remaining = size; + cur->node = block; + break; + case TTM_PL_TT: + node = to_ttm_range_mgr_node(res)->mm_nodes; + while (start >= node->size << PAGE_SHIFT) + start -= node++->size << PAGE_SHIFT; + + cur->start = (node->start << PAGE_SHIFT) + start; + cur->size = min((node->size << PAGE_SHIFT) - start, size); + cur->remaining = size; + cur->node = node; + break; + default: + goto err_out; + }
- cur->start = (node->start << PAGE_SHIFT) + start; - cur->size = min((node->size << PAGE_SHIFT) - start, size); + return; + +err_out: + cur->start = start; + cur->size = size; cur->remaining = size; - cur->node = node; + cur->node = NULL; + WARN_ON(res && start + size > res->num_pages << PAGE_SHIFT); + return; }
/** @@ -85,7 +124,9 @@ static inline void amdgpu_res_first(struct ttm_resource *res, */ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size) { - struct drm_mm_node *node = cur->node; + struct drm_buddy_block *block; + struct drm_mm_node *node; + struct list_head *next;
BUG_ON(size > cur->remaining);
@@ -99,9 +140,27 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size) return; }
- cur->node = ++node; - cur->start = node->start << PAGE_SHIFT; - cur->size = min(node->size << PAGE_SHIFT, cur->remaining); + switch (cur->mem_type) { + case TTM_PL_VRAM: + block = cur->node; + + next = block->link.next; + block = list_entry(next, struct drm_buddy_block, link); + + cur->node = block; + cur->start = amdgpu_node_start(block); + cur->size = min(amdgpu_node_size(block), cur->remaining); + break; + case TTM_PL_TT: + node = cur->node; + + cur->node = ++node; + cur->start = node->start << PAGE_SHIFT; + cur->size = min(node->size << PAGE_SHIFT, cur->remaining); + break; + default: + return; + } }
#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index f8f48be16d80..1a5cbd5ef738 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -26,6 +26,7 @@
#include <linux/dma-direction.h> #include <drm/gpu_scheduler.h> +#include <drm/drm_buddy.h> #include "amdgpu.h"
#define AMDGPU_PL_GDS (TTM_PL_PRIV + 0) @@ -40,12 +41,14 @@
struct amdgpu_vram_mgr { struct ttm_resource_manager manager; - struct drm_mm mm; - spinlock_t lock; + struct drm_buddy mm; + /* protects access to buffer objects */ + struct mutex lock; struct list_head reservations_pending; struct list_head reserved_pages; atomic64_t usage; atomic64_t vis_usage; + u64 default_page_size; };
struct amdgpu_gtt_mgr { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 7a2b487db57c..b7c597183317 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -32,8 +32,11 @@ #include "atom.h"
struct amdgpu_vram_reservation { + u64 start; + u64 size; + unsigned long flags; + struct list_head block; struct list_head node; - struct drm_mm_node mm_node; };
static inline struct amdgpu_vram_mgr * @@ -194,10 +197,10 @@ const struct attribute_group amdgpu_vram_mgr_attr_group = { * Calculate how many bytes of the MM node are inside visible VRAM */ static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, - struct drm_mm_node *node) + struct drm_buddy_block *block) { - uint64_t start = node->start << PAGE_SHIFT; - uint64_t end = (node->size + node->start) << PAGE_SHIFT; + u64 start = amdgpu_node_start(block); + u64 end = start + amdgpu_node_size(block);
if (start >= adev->gmc.visible_vram_size) return 0; @@ -218,9 +221,9 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct ttm_resource *res = bo->tbo.resource; - unsigned pages = res->num_pages; - struct drm_mm_node *mm; - u64 usage; + struct amdgpu_vram_mgr_node *node = to_amdgpu_vram_mgr_node(res); + struct drm_buddy_block *block; + u64 usage = 0;
if (amdgpu_gmc_vram_full_visible(&adev->gmc)) return amdgpu_bo_size(bo); @@ -228,9 +231,8 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) return 0;
- mm = &container_of(res, struct ttm_range_mgr_node, base)->mm_nodes[0]; - for (usage = 0; pages; pages -= mm->size, mm++) - usage += amdgpu_vram_mgr_vis_size(adev, mm); + list_for_each_entry(block, &node->blocks, link) + usage += amdgpu_vram_mgr_vis_size(adev, block);
return usage; } @@ -240,21 +242,29 @@ static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) { struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); - struct drm_mm *mm = &mgr->mm; + struct drm_buddy *mm = &mgr->mm; struct amdgpu_vram_reservation *rsv, *temp; + struct drm_buddy_block *block; uint64_t vis_usage;
list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) { - if (drm_mm_reserve_node(mm, &rsv->mm_node)) + if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, + rsv->size, mm->chunk_size, &rsv->block, + rsv->flags)) continue;
- dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", - rsv->mm_node.start, rsv->mm_node.size); - - vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node); - atomic64_add(vis_usage, &mgr->vis_usage); - atomic64_add(rsv->mm_node.size << PAGE_SHIFT, &mgr->usage); - list_move(&rsv->node, &mgr->reserved_pages); + block = list_first_entry_or_null(&rsv->block, + struct drm_buddy_block, + link); + if (block) { + dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", + rsv->start, rsv->size); + + vis_usage = amdgpu_vram_mgr_vis_size(adev, block); + atomic64_add(vis_usage, &mgr->vis_usage); + atomic64_add(rsv->size, &mgr->usage); + list_move(&rsv->node, &mgr->reserved_pages); + } } }
@@ -277,13 +287,16 @@ int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, return -ENOMEM;
INIT_LIST_HEAD(&rsv->node); - rsv->mm_node.start = start >> PAGE_SHIFT; - rsv->mm_node.size = size >> PAGE_SHIFT; + INIT_LIST_HEAD(&rsv->block); + + rsv->start = start; + rsv->size = size; + rsv->flags |= DRM_BUDDY_RANGE_ALLOCATION;
- spin_lock(&mgr->lock); - list_add_tail(&mgr->reservations_pending, &rsv->node); + mutex_lock(&mgr->lock); + list_add_tail(&rsv->node, &mgr->reservations_pending); amdgpu_vram_mgr_do_reserve(&mgr->manager); - spin_unlock(&mgr->lock); + mutex_unlock(&mgr->lock);
return 0; } @@ -305,19 +318,19 @@ int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, struct amdgpu_vram_reservation *rsv; int ret;
- spin_lock(&mgr->lock); + mutex_lock(&mgr->lock);
list_for_each_entry(rsv, &mgr->reservations_pending, node) { - if ((rsv->mm_node.start <= start) && - (start < (rsv->mm_node.start + rsv->mm_node.size))) { + if (rsv->start <= start && + (start < (rsv->start + rsv->size))) { ret = -EBUSY; goto out; } }
list_for_each_entry(rsv, &mgr->reserved_pages, node) { - if ((rsv->mm_node.start <= start) && - (start < (rsv->mm_node.start + rsv->mm_node.size))) { + if (rsv->start <= start && + (start < (rsv->start + rsv->size))) { ret = 0; goto out; } @@ -325,32 +338,10 @@ int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
ret = -ENOENT; out: - spin_unlock(&mgr->lock); + mutex_unlock(&mgr->lock); return ret; }
-/** - * amdgpu_vram_mgr_virt_start - update virtual start address - * - * @mem: ttm_resource to update - * @node: just allocated node - * - * Calculate a virtual BO start address to easily check if everything is CPU - * accessible. - */ -static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem, - struct drm_mm_node *node) -{ - unsigned long start; - - start = node->start + node->size; - if (start > mem->num_pages) - start -= mem->num_pages; - else - start = 0; - mem->start = max(mem->start, start); -} - /** * amdgpu_vram_mgr_new - allocate new ranges * @@ -366,13 +357,13 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, const struct ttm_place *place, struct ttm_resource **res) { - unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages; + unsigned long lpfn, pages_per_node, pages_left, pages, n_pages; + u64 vis_usage = 0, mem_bytes, max_bytes, min_page_size; struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); - uint64_t vis_usage = 0, mem_bytes, max_bytes; - struct ttm_range_mgr_node *node; - struct drm_mm *mm = &mgr->mm; - enum drm_mm_insert_mode mode; + struct amdgpu_vram_mgr_node *node; + struct drm_buddy *mm = &mgr->mm; + struct drm_buddy_block *block; unsigned i; int r;
@@ -391,10 +382,9 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, goto error_sub; }
- if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) pages_per_node = ~0ul; - num_nodes = 1; - } else { + else { #ifdef CONFIG_TRANSPARENT_HUGEPAGE pages_per_node = HPAGE_PMD_NR; #else @@ -403,11 +393,9 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, #endif pages_per_node = max_t(uint32_t, pages_per_node, tbo->page_alignment); - num_nodes = DIV_ROUND_UP_ULL(PFN_UP(mem_bytes), pages_per_node); }
- node = kvmalloc(struct_size(node, mm_nodes, num_nodes), - GFP_KERNEL | __GFP_ZERO); + node = kzalloc(sizeof(*node), GFP_KERNEL); if (!node) { r = -ENOMEM; goto error_sub; @@ -415,9 +403,17 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
ttm_resource_init(tbo, place, &node->base);
- mode = DRM_MM_INSERT_BEST; + INIT_LIST_HEAD(&node->blocks); + if (place->flags & TTM_PL_FLAG_TOPDOWN) - mode = DRM_MM_INSERT_HIGH; + node->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + + if (place->fpfn || lpfn != man->size) + /* Allocate blocks in desired range */ + node->flags |= DRM_BUDDY_RANGE_ALLOCATION; + + min_page_size = mgr->default_page_size; + BUG_ON(min_page_size < mm->chunk_size);
pages_left = node->base.num_pages;
@@ -425,36 +421,61 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, pages = min(pages_left, 2UL << (30 - PAGE_SHIFT));
i = 0; - spin_lock(&mgr->lock); while (pages_left) { - uint32_t alignment = tbo->page_alignment; - if (pages >= pages_per_node) - alignment = pages_per_node; - - r = drm_mm_insert_node_in_range(mm, &node->mm_nodes[i], pages, - alignment, 0, place->fpfn, - lpfn, mode); - if (unlikely(r)) { - if (pages > pages_per_node) { - if (is_power_of_2(pages)) - pages = pages / 2; - else - pages = rounddown_pow_of_two(pages); - continue; - } - goto error_free; + pages = pages_per_node; + + n_pages = pages; + + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + n_pages = roundup_pow_of_two(n_pages); + min_page_size = (u64)n_pages << PAGE_SHIFT; + + if (n_pages > lpfn) + lpfn = n_pages; }
- vis_usage += amdgpu_vram_mgr_vis_size(adev, &node->mm_nodes[i]); - amdgpu_vram_mgr_virt_start(&node->base, &node->mm_nodes[i]); + mutex_lock(&mgr->lock); + r = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, + (u64)lpfn << PAGE_SHIFT, + (u64)n_pages << PAGE_SHIFT, + min_page_size, + &node->blocks, + node->flags); + mutex_unlock(&mgr->lock); + if (unlikely(r)) + goto error_free_blocks; + pages_left -= pages; ++i;
if (pages > pages_left) pages = pages_left; } - spin_unlock(&mgr->lock); + + /* Free unused pages for contiguous allocation */ + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + u64 actual_size = (u64)node->base.num_pages << PAGE_SHIFT; + + mutex_lock(&mgr->lock); + drm_buddy_block_trim(mm, + actual_size, + &node->blocks); + mutex_unlock(&mgr->lock); + } + + list_for_each_entry(block, &node->blocks, link) + vis_usage += amdgpu_vram_mgr_vis_size(adev, block); + + block = list_first_entry_or_null(&node->blocks, + struct drm_buddy_block, + link); + if (!block) { + r = -ENOENT; + goto error_free_res; + } + + node->base.start = amdgpu_node_start(block) >> PAGE_SHIFT;
if (i == 1) node->base.placement |= TTM_PL_FLAG_CONTIGUOUS; @@ -468,12 +489,12 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, *res = &node->base; return 0;
-error_free: - while (i--) - drm_mm_remove_node(&node->mm_nodes[i]); - spin_unlock(&mgr->lock); - kvfree(node); - +error_free_blocks: + mutex_lock(&mgr->lock); + drm_buddy_free_list(mm, &node->blocks); + mutex_unlock(&mgr->lock); +error_free_res: + kfree(node); error_sub: atomic64_sub(mem_bytes, &mgr->usage); return r; @@ -490,28 +511,28 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, struct ttm_resource *res) { - struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res); + struct amdgpu_vram_mgr_node *node = to_amdgpu_vram_mgr_node(res); struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); + struct drm_buddy *mm = &mgr->mm; + struct drm_buddy_block *block; uint64_t usage = 0, vis_usage = 0; - unsigned i, pages;
- spin_lock(&mgr->lock); - for (i = 0, pages = res->num_pages; pages; - pages -= node->mm_nodes[i].size, ++i) { - struct drm_mm_node *mm = &node->mm_nodes[i]; - - drm_mm_remove_node(mm); - usage += mm->size << PAGE_SHIFT; - vis_usage += amdgpu_vram_mgr_vis_size(adev, mm); + mutex_lock(&mgr->lock); + list_for_each_entry(block, &node->blocks, link) { + usage += amdgpu_node_size(block); + vis_usage += amdgpu_vram_mgr_vis_size(adev, block); } + amdgpu_vram_mgr_do_reserve(man); - spin_unlock(&mgr->lock); + + drm_buddy_free_list(mm, &node->blocks); + mutex_unlock(&mgr->lock);
atomic64_sub(usage, &mgr->usage); atomic64_sub(vis_usage, &mgr->vis_usage);
- kvfree(node); + kfree(node); }
/** @@ -661,10 +682,19 @@ static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, struct drm_printer *printer) { struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); + struct drm_buddy *mm = &mgr->mm; + struct drm_buddy_block *block; + + mutex_lock(&mgr->lock); + drm_printf(printer, "default_page_size: %lluKiB\n", + mgr->default_page_size >> 10);
- spin_lock(&mgr->lock); - drm_mm_print(&mgr->mm, printer); - spin_unlock(&mgr->lock); + drm_buddy_print(mm, printer); + + drm_printf(printer, "reserved:\n"); + list_for_each_entry(block, &mgr->reserved_pages, link) + drm_buddy_block_print(mm, block, printer); + mutex_unlock(&mgr->lock);
drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", man->size, amdgpu_vram_mgr_usage(mgr) >> 20, @@ -688,15 +718,20 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) { struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; struct ttm_resource_manager *man = &mgr->manager; + int err;
ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT);
man->func = &amdgpu_vram_mgr_func;
- drm_mm_init(&mgr->mm, 0, man->size); - spin_lock_init(&mgr->lock); + err = drm_buddy_init(&mgr->mm, man->size << PAGE_SHIFT, PAGE_SIZE); + if (err) + return err; + + mutex_init(&mgr->lock); INIT_LIST_HEAD(&mgr->reservations_pending); INIT_LIST_HEAD(&mgr->reserved_pages); + mgr->default_page_size = PAGE_SIZE;
ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); ttm_resource_manager_set_used(man, true); @@ -724,16 +759,16 @@ void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) if (ret) return;
- spin_lock(&mgr->lock); + mutex_lock(&mgr->lock); list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) kfree(rsv);
list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) { - drm_mm_remove_node(&rsv->mm_node); + drm_buddy_free_list(&mgr->mm, &rsv->block); kfree(rsv); } - drm_mm_takedown(&mgr->mm); - spin_unlock(&mgr->lock); + drm_buddy_fini(&mgr->mm); + mutex_unlock(&mgr->lock);
ttm_resource_manager_cleanup(man); ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
On 19/01/2022 11:37, Arunpravin wrote:
Make drm_buddy_alloc a single function to handle range allocation and non-range allocation demands
Implemented a new function alloc_range() which allocates the requested power-of-two block comply with range limitations
Moved order computation and memory alignment logic from i915 driver to drm buddy
v2: merged below changes to keep the build unbroken - drm_buddy_alloc_range() becomes obsolete and may be removed - enable ttm range allocation (fpfn / lpfn) support in i915 driver - apply enhanced drm_buddy_alloc() function to i915 driver
v3(Matthew Auld):
- Fix alignment issues and remove unnecessary list_empty check
- add more validation checks for input arguments
- make alloc_range() block allocations as bottom-up
- optimize order computation logic
- replace uint64_t with u64, which is preferred in the kernel
v4(Matthew Auld):
- keep drm_buddy_alloc_range() function implementation for generic actual range allocations
- keep alloc_range() implementation for end bias allocations
v5(Matthew Auld):
- modify drm_buddy_alloc() passing argument place->lpfn to lpfn as place->lpfn will currently always be zero for i915
v6(Matthew Auld):
- fixup potential uaf - If we are unlucky and can't allocate enough memory when splitting blocks, where we temporarily end up with the given block and its buddy on the respective free list, then we need to ensure we delete both blocks, and no just the buddy, before potentially freeing them
Hmm, not sure we really want to squash existing bug fixes into this patch. Perhaps bring in [1] to the start of your series? i915_buddy is gone now. Alternatively I can resend such that it applies on top drm_buddy. Your choice.
[1] https://patchwork.freedesktop.org/patch/469806/?series=98953&rev=1
- fix warnings reported by kernel test robot lkp@intel.com
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com
drivers/gpu/drm/drm_buddy.c | 326 +++++++++++++----- drivers/gpu/drm/i915/i915_ttm_buddy_manager.c | 67 ++-- drivers/gpu/drm/i915/i915_ttm_buddy_manager.h | 2 + include/drm/drm_buddy.h | 22 +- 4 files changed, 293 insertions(+), 124 deletions(-)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index d60878bc9c20..954e31962c74 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -282,23 +282,99 @@ void drm_buddy_free_list(struct drm_buddy *mm, struct list_head *objects) } EXPORT_SYMBOL(drm_buddy_free_list);
-/**
- drm_buddy_alloc_blocks - allocate power-of-two blocks
- @mm: DRM buddy manager to allocate from
- @order: size of the allocation
- The order value here translates to:
- 0 = 2^0 * mm->chunk_size
- 1 = 2^1 * mm->chunk_size
- 2 = 2^2 * mm->chunk_size
- Returns:
- allocated ptr to the &drm_buddy_block on success
- */
-struct drm_buddy_block * -drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order) +static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) +{
- return s1 <= e2 && e1 >= s2;
+}
+static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) +{
- return s1 <= s2 && e1 >= e2;
+}
+static struct drm_buddy_block * +alloc_range_bias(struct drm_buddy *mm,
u64 start, u64 end,unsigned int order)+{
- struct drm_buddy_block *block;
- struct drm_buddy_block *buddy;
- LIST_HEAD(dfs);
- int err;
- int i;
- end = end - 1;
- for (i = 0; i < mm->n_roots; ++i)
list_add_tail(&mm->roots[i]->tmp_link, &dfs);- do {
u64 block_start;u64 block_end;block = list_first_entry_or_null(&dfs,struct drm_buddy_block,tmp_link);if (!block)break;list_del(&block->tmp_link);if (drm_buddy_block_order(block) < order)continue;block_start = drm_buddy_block_offset(block);block_end = block_start + drm_buddy_block_size(mm, block) - 1;if (!overlaps(start, end, block_start, block_end))continue;if (drm_buddy_block_is_allocated(block))continue;if (contains(start, end, block_start, block_end) &&order == drm_buddy_block_order(block)) {/** Find the free block within the range.*/if (drm_buddy_block_is_free(block))return block;continue;}if (!drm_buddy_block_is_split(block)) {err = split_block(mm, block);if (unlikely(err))goto err_undo;}list_add(&block->right->tmp_link, &dfs);list_add(&block->left->tmp_link, &dfs);- } while (1);
- return ERR_PTR(-ENOSPC);
+err_undo:
- /*
* We really don't want to leave around a bunch of split blocks, since* bigger is better, so make sure we merge everything back before we* free the allocated blocks.*/- buddy = get_buddy(block);
- if (buddy &&
(drm_buddy_block_is_free(block) &&drm_buddy_block_is_free(buddy))) {list_del(&block->link);__drm_buddy_free(mm, block);- }
- return ERR_PTR(err);
+}
+static struct drm_buddy_block * +alloc_from_freelist(struct drm_buddy *mm,
unsigned int order,unsigned long flags)@@ -320,78 +396,30 @@ drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order) while (i != order) { err = split_block(mm, block); if (unlikely(err))
goto out_free;
goto err_undo;
/* Go low */block = block->left;
block = block->right;
- mark_allocated(block);
- mm->avail -= drm_buddy_block_size(mm, block);
- kmemleak_update_trace(block); return block;
-out_free:
- if (i != order)
+err_undo:
- if (i != order) {
list_del(&block->link);- } return ERR_PTR(err); }
-EXPORT_SYMBOL(drm_buddy_alloc_blocks);
-static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) -{
- return s1 <= e2 && e1 >= s2;
-}
-static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) -{
- return s1 <= s2 && e1 >= e2;
-}
-/**
- drm_buddy_alloc_range - allocate range
- @mm: DRM buddy manager to allocate from
- @blocks: output list head to add allocated blocks
- @start: start of the allowed range for this block
- @size: size of the allocation
- Intended for pre-allocating portions of the address space, for example to
- reserve a block for the initial framebuffer or similar, hence the expectation
- here is that drm_buddy_alloc_blocks() is still the main vehicle for
- allocations, so if that's not the case then the drm_mm range allocator is
- probably a much better fit, and so you should probably go use that instead.
- Note that it's safe to chain together multiple alloc_ranges
- with the same blocks list
- Returns:
- 0 on success, error code on failure.
- */
-int drm_buddy_alloc_range(struct drm_buddy *mm,
struct list_head *blocks,u64 start, u64 size)+static int __alloc_range(struct drm_buddy *mm,
struct list_head *dfs,u64 start, u64 size,struct list_head *blocks)
LIST_HEAD(allocated);
LIST_HEAD(dfs); u64 end; int err;
int i;
if (size < mm->chunk_size)
return -EINVAL;if (!IS_ALIGNED(size | start, mm->chunk_size))
return -EINVAL;if (range_overflows(start, size, mm->size))
return -EINVAL;for (i = 0; i < mm->n_roots; ++i)
list_add_tail(&mm->roots[i]->tmp_link, &dfs);end = start + size - 1;
@@ -399,7 +427,7 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, u64 block_start; u64 block_end;
block = list_first_entry_or_null(&dfs,
block = list_first_entry_or_null(dfs, struct drm_buddy_block, tmp_link);@@ -426,7 +454,7 @@ int drm_buddy_alloc_range(struct drm_buddy *mm,
mark_allocated(block); mm->avail -= drm_buddy_block_size(mm, block);
list_add_tail(&block->link, &allocated);
list_add_tail(&block->link, blocks); continue;@@ -436,11 +464,10 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, goto err_undo; }
list_add(&block->right->tmp_link, &dfs);list_add(&block->left->tmp_link, &dfs);
list_add(&block->right->tmp_link, dfs);list_add(&block->left->tmp_link, dfs);
- list_splice_tail(&allocated, blocks);
Maybe keep this. That way it will be consistent with what drm_buddy_alloc_blocks is doing? Otherwise if there is a good reason for dropping this, then we should also make drm_buddy_alloc_blocks do the same thing. At the moment if chaining allocations with the same list, this will also nuke the existing nodes if we ever hit err_free, but the other path leaves the existing nodes untouched, AFAIK.
return 0;
err_undo: @@ -452,14 +479,149 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, buddy = get_buddy(block); if (buddy && (drm_buddy_block_is_free(block) &&
drm_buddy_block_is_free(buddy)))
drm_buddy_block_is_free(buddy))) {list_del(&block->link);- }
+err_free:
- drm_buddy_free_list(mm, blocks);
- return err;
+}
+/**
- __drm_buddy_alloc_range - actual range allocation
- @mm: DRM buddy manager to allocate from
- @start: start of the allowed range for this block
- @size: size of the allocation
- @blocks: output list head to add allocated blocks
- Intended for pre-allocating portions of the address space, for example to
- reserve a block for the initial framebuffer or similar
- Note that it's safe to chain together multiple alloc_ranges
- with the same blocks list
- Returns:
- 0 on success, error code on failure.
- */
+static int __drm_buddy_alloc_range(struct drm_buddy *mm,
u64 start,u64 size,struct list_head *blocks)+{
- LIST_HEAD(dfs);
- int i;
- for (i = 0; i < mm->n_roots; ++i)
list_add_tail(&mm->roots[i]->tmp_link, &dfs);- return __alloc_range(mm, &dfs, start, size, blocks);
+}
+/**
- drm_buddy_alloc_blocks - allocate power-of-two blocks
- @mm: DRM buddy manager to allocate from
- @start: start of the allowed range for this block
- @end: end of the allowed range for this block
- @size: size of the allocation
- @min_page_size: alignment of the allocation
- @blocks: output list head to add allocated blocks
- @flags: DRM_BUDDY_*_ALLOCATION flags
- alloc_range_bias() called on range limitations, which traverses
- the tree and returns the desired block.
- alloc_from_freelist() called when *no* range restrictions
- are enforced, which picks the block from the freelist.
- blocks are allocated in order, the order value here translates to:
- 0 = 2^0 * mm->chunk_size
- 1 = 2^1 * mm->chunk_size
- 2 = 2^2 * mm->chunk_size
- Returns:
- 0 on success, error code on failure.
- */
+int drm_buddy_alloc_blocks(struct drm_buddy *mm,
u64 start, u64 end, u64 size,u64 min_page_size,struct list_head *blocks,unsigned long flags)+{
struct drm_buddy_block *block = NULL;
unsigned int min_order, order;
unsigned long pages;
LIST_HEAD(allocated);
int err;
if (size < mm->chunk_size)
return -EINVAL;if (min_page_size < mm->chunk_size)
return -EINVAL;if (!is_power_of_2(min_page_size))
return -EINVAL;if (!IS_ALIGNED(start | end | size, mm->chunk_size))
return -EINVAL;if (check_range_overflow(start, end, size, mm->size))
return -EINVAL;/* Actual range allocation */
if (start + size == end)
return __drm_buddy_alloc_range(mm, start, size, blocks);pages = size >> ilog2(mm->chunk_size);
order = fls(pages) - 1;
min_order = ilog2(min_page_size) - ilog2(mm->chunk_size);
do {
order = min(order, (unsigned int)fls(pages) - 1);BUG_ON(order > mm->max_order);BUG_ON(order < min_order);do {if (flags & DRM_BUDDY_RANGE_ALLOCATION)/* Allocate traversing within the range */block = alloc_range_bias(mm, start, end, order);else/* Allocate from freelist */block = alloc_from_freelist(mm, order, flags);if (!IS_ERR(block))break;if (order-- == min_order) {err = -ENOSPC;goto err_free;}} while (1);mark_allocated(block);mm->avail -= drm_buddy_block_size(mm, block);kmemleak_update_trace(block);list_add_tail(&block->link, &allocated);pages -= BIT(order);if (!pages)break;} while (1);
list_splice_tail(&allocated, blocks);
return 0;
err_free: drm_buddy_free_list(mm, &allocated); return err; }
-EXPORT_SYMBOL(drm_buddy_alloc_range); +EXPORT_SYMBOL(drm_buddy_alloc_blocks);
/**
- drm_buddy_block_print - print block information
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 6ba314f9836a..1411f4cf1f21 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -36,13 +36,14 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct i915_ttm_buddy_resource *bman_res; struct drm_buddy *mm = &bman->mm;
- unsigned long n_pages;
- unsigned int min_order;
- unsigned long n_pages, lpfn; u64 min_page_size; u64 size; int err;
- GEM_BUG_ON(place->fpfn || place->lpfn);
lpfn = place->lpfn;
if (!lpfn)
lpfn = man->size;bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL); if (!bman_res)
@@ -52,6 +53,9 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, INIT_LIST_HEAD(&bman_res->blocks); bman_res->mm = mm;
- if (place->fpfn || lpfn != man->size)
bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;- GEM_BUG_ON(!bman_res->base.num_pages); size = bman_res->base.num_pages << PAGE_SHIFT;
@@ -60,10 +64,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, min_page_size = bo->page_alignment << PAGE_SHIFT;
GEM_BUG_ON(min_page_size < mm->chunk_size);
- min_order = ilog2(min_page_size) - ilog2(mm->chunk_size);
- if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
unsigned long pages;- size = roundup_pow_of_two(size);
min_order = ilog2(size) - ilog2(mm->chunk_size);
min_page_size = size;pages = size >> ilog2(mm->chunk_size);if (pages > lpfn)lpfn = pages;}
if (size > mm->size) {
@@ -73,34 +83,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
n_pages = size >> ilog2(mm->chunk_size);
- do {
struct drm_buddy_block *block;unsigned int order;order = fls(n_pages) - 1;GEM_BUG_ON(order > mm->max_order);GEM_BUG_ON(order < min_order);do {mutex_lock(&bman->lock);block = drm_buddy_alloc_blocks(mm, order);mutex_unlock(&bman->lock);if (!IS_ERR(block))break;if (order-- == min_order) {err = -ENOSPC;goto err_free_blocks;}} while (1);n_pages -= BIT(order);list_add_tail(&block->link, &bman_res->blocks);if (!n_pages)break;- } while (1);
mutex_lock(&bman->lock);
err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
(u64)lpfn << PAGE_SHIFT,(u64)n_pages << PAGE_SHIFT,min_page_size,&bman_res->blocks,bman_res->flags);mutex_unlock(&bman->lock);
if (unlikely(err))
goto err_free_blocks;*res = &bman_res->base; return 0;
@@ -266,10 +258,17 @@ int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct drm_buddy *mm = &bman->mm;
unsigned long flags = 0; int ret;
flags |= DRM_BUDDY_RANGE_ALLOCATION;
mutex_lock(&bman->lock);
- ret = drm_buddy_alloc_range(mm, &bman->reserved, start, size);
ret = drm_buddy_alloc_blocks(mm, start,
start + size,size, mm->chunk_size,&bman->reserved,flags);mutex_unlock(&bman->lock);
return ret;
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h index 312077941411..72c90b432e87 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h @@ -20,6 +20,7 @@ struct drm_buddy;
- @base: struct ttm_resource base class we extend
- @blocks: the list of struct i915_buddy_block for this resource/allocation
- @flags: DRM_BUDDY_*_ALLOCATION flags
- @mm: the struct i915_buddy_mm for this resource
- Extends the struct ttm_resource to manage an address space allocation with
@@ -28,6 +29,7 @@ struct drm_buddy; struct i915_ttm_buddy_resource { struct ttm_resource base; struct list_head blocks;
- unsigned long flags; struct drm_buddy *mm; };
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index f524db152413..865664b90a8a 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -13,15 +13,22 @@
#include <drm/drm_print.h>
-#define range_overflows(start, size, max) ({ \ +#define check_range_overflow(start, end, size, max) ({ \ typeof(start) start__ = (start); \
- typeof(end) end__ = (end);\ typeof(size) size__ = (size); \ typeof(max) max__ = (max); \ (void)(&start__ == &size__); \ (void)(&start__ == &max__); \
- start__ >= max__ || size__ > max__ - start__; \
- (void)(&start__ == &end__); \
- (void)(&end__ == &size__); \
- (void)(&end__ == &max__); \
- start__ >= max__ || end__ > max__ || \
- size__ > end__ - start__; \
Hmm, this doesn't say anything about start >= end. IIUC this will incorrectly still allow something like:
check_range_overflow(PAGE_SIZE, 0ULL, PAGE_SIZE, mm->size)
Perhaps just keep the macro as it was, and then maybe just add something like:
if (end > mm->size) return -EINVAL;
if (range_overflows(start, size, end)) return -EINVAL;
IMO that looks a little easier to understand?
Otherwise I think looks good.
})
+#define DRM_BUDDY_RANGE_ALLOCATION (1 << 0)
- struct drm_buddy_block { #define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) #define DRM_BUDDY_HEADER_STATE GENMASK_ULL(11, 10)
@@ -131,12 +138,11 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size);
void drm_buddy_fini(struct drm_buddy *mm);
-struct drm_buddy_block * -drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order);
-int drm_buddy_alloc_range(struct drm_buddy *mm,
struct list_head *blocks,u64 start, u64 size);+int drm_buddy_alloc_blocks(struct drm_buddy *mm,
u64 start, u64 end, u64 size,u64 min_page_size,struct list_head *blocks,unsigned long flags);void drm_buddy_free_block(struct drm_buddy *mm, struct drm_buddy_block *block);
base-commit: e3a3896bf9a189f3992db9fd8622893cf90652cb
On 21/01/22 5:30 pm, Matthew Auld wrote:
On 19/01/2022 11:37, Arunpravin wrote:
Make drm_buddy_alloc a single function to handle range allocation and non-range allocation demands
Implemented a new function alloc_range() which allocates the requested power-of-two block comply with range limitations
Moved order computation and memory alignment logic from i915 driver to drm buddy
v2: merged below changes to keep the build unbroken - drm_buddy_alloc_range() becomes obsolete and may be removed - enable ttm range allocation (fpfn / lpfn) support in i915 driver - apply enhanced drm_buddy_alloc() function to i915 driver
v3(Matthew Auld):
- Fix alignment issues and remove unnecessary list_empty check
- add more validation checks for input arguments
- make alloc_range() block allocations as bottom-up
- optimize order computation logic
- replace uint64_t with u64, which is preferred in the kernel
v4(Matthew Auld):
- keep drm_buddy_alloc_range() function implementation for generic actual range allocations
- keep alloc_range() implementation for end bias allocations
v5(Matthew Auld):
- modify drm_buddy_alloc() passing argument place->lpfn to lpfn as place->lpfn will currently always be zero for i915
v6(Matthew Auld):
- fixup potential uaf - If we are unlucky and can't allocate enough memory when splitting blocks, where we temporarily end up with the given block and its buddy on the respective free list, then we need to ensure we delete both blocks, and no just the buddy, before potentially freeing them
Hmm, not sure we really want to squash existing bug fixes into this patch. Perhaps bring in [1] to the start of your series? i915_buddy is gone now. Alternatively I can resend such that it applies on top drm_buddy. Your choice.
[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatchwork....
I will revert this fix in v10, please resend on top drm_buddy
- fix warnings reported by kernel test robot lkp@intel.com
Signed-off-by: Arunpravin Arunpravin.PaneerSelvam@amd.com
drivers/gpu/drm/drm_buddy.c | 326 +++++++++++++----- drivers/gpu/drm/i915/i915_ttm_buddy_manager.c | 67 ++-- drivers/gpu/drm/i915/i915_ttm_buddy_manager.h | 2 + include/drm/drm_buddy.h | 22 +- 4 files changed, 293 insertions(+), 124 deletions(-)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index d60878bc9c20..954e31962c74 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -282,23 +282,99 @@ void drm_buddy_free_list(struct drm_buddy *mm, struct list_head *objects) } EXPORT_SYMBOL(drm_buddy_free_list);
-/**
- drm_buddy_alloc_blocks - allocate power-of-two blocks
- @mm: DRM buddy manager to allocate from
- @order: size of the allocation
- The order value here translates to:
- 0 = 2^0 * mm->chunk_size
- 1 = 2^1 * mm->chunk_size
- 2 = 2^2 * mm->chunk_size
- Returns:
- allocated ptr to the &drm_buddy_block on success
- */
-struct drm_buddy_block * -drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order) +static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) +{
- return s1 <= e2 && e1 >= s2;
+}
+static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) +{
- return s1 <= s2 && e1 >= e2;
+}
+static struct drm_buddy_block * +alloc_range_bias(struct drm_buddy *mm,
u64 start, u64 end,unsigned int order)+{
- struct drm_buddy_block *block;
- struct drm_buddy_block *buddy;
- LIST_HEAD(dfs);
- int err;
- int i;
- end = end - 1;
- for (i = 0; i < mm->n_roots; ++i)
list_add_tail(&mm->roots[i]->tmp_link, &dfs);- do {
u64 block_start;u64 block_end;block = list_first_entry_or_null(&dfs,struct drm_buddy_block,tmp_link);if (!block)break;list_del(&block->tmp_link);if (drm_buddy_block_order(block) < order)continue;block_start = drm_buddy_block_offset(block);block_end = block_start + drm_buddy_block_size(mm, block) - 1;if (!overlaps(start, end, block_start, block_end))continue;if (drm_buddy_block_is_allocated(block))continue;if (contains(start, end, block_start, block_end) &&order == drm_buddy_block_order(block)) {/** Find the free block within the range.*/if (drm_buddy_block_is_free(block))return block;continue;}if (!drm_buddy_block_is_split(block)) {err = split_block(mm, block);if (unlikely(err))goto err_undo;}list_add(&block->right->tmp_link, &dfs);list_add(&block->left->tmp_link, &dfs);- } while (1);
- return ERR_PTR(-ENOSPC);
+err_undo:
- /*
* We really don't want to leave around a bunch of split blocks, since* bigger is better, so make sure we merge everything back before we* free the allocated blocks.*/- buddy = get_buddy(block);
- if (buddy &&
(drm_buddy_block_is_free(block) &&drm_buddy_block_is_free(buddy))) {list_del(&block->link);__drm_buddy_free(mm, block);- }
- return ERR_PTR(err);
+}
+static struct drm_buddy_block * +alloc_from_freelist(struct drm_buddy *mm,
unsigned int order,unsigned long flags)@@ -320,78 +396,30 @@ drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order) while (i != order) { err = split_block(mm, block); if (unlikely(err))
goto out_free;
goto err_undo;
/* Go low */block = block->left;
block = block->right;
- mark_allocated(block);
- mm->avail -= drm_buddy_block_size(mm, block);
- kmemleak_update_trace(block); return block;
-out_free:
- if (i != order)
+err_undo:
- if (i != order) {
list_del(&block->link);- } return ERR_PTR(err); }
-EXPORT_SYMBOL(drm_buddy_alloc_blocks);
-static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) -{
- return s1 <= e2 && e1 >= s2;
-}
-static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) -{
- return s1 <= s2 && e1 >= e2;
-}
-/**
- drm_buddy_alloc_range - allocate range
- @mm: DRM buddy manager to allocate from
- @blocks: output list head to add allocated blocks
- @start: start of the allowed range for this block
- @size: size of the allocation
- Intended for pre-allocating portions of the address space, for example to
- reserve a block for the initial framebuffer or similar, hence the expectation
- here is that drm_buddy_alloc_blocks() is still the main vehicle for
- allocations, so if that's not the case then the drm_mm range allocator is
- probably a much better fit, and so you should probably go use that instead.
- Note that it's safe to chain together multiple alloc_ranges
- with the same blocks list
- Returns:
- 0 on success, error code on failure.
- */
-int drm_buddy_alloc_range(struct drm_buddy *mm,
struct list_head *blocks,u64 start, u64 size)+static int __alloc_range(struct drm_buddy *mm,
struct list_head *dfs,u64 start, u64 size,struct list_head *blocks)
LIST_HEAD(allocated);
LIST_HEAD(dfs); u64 end; int err;
int i;
if (size < mm->chunk_size)
return -EINVAL;if (!IS_ALIGNED(size | start, mm->chunk_size))
return -EINVAL;if (range_overflows(start, size, mm->size))
return -EINVAL;for (i = 0; i < mm->n_roots; ++i)
list_add_tail(&mm->roots[i]->tmp_link, &dfs);end = start + size - 1;
@@ -399,7 +427,7 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, u64 block_start; u64 block_end;
block = list_first_entry_or_null(&dfs,
block = list_first_entry_or_null(dfs, struct drm_buddy_block, tmp_link);@@ -426,7 +454,7 @@ int drm_buddy_alloc_range(struct drm_buddy *mm,
mark_allocated(block); mm->avail -= drm_buddy_block_size(mm, block);
list_add_tail(&block->link, &allocated);
list_add_tail(&block->link, blocks); continue;@@ -436,11 +464,10 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, goto err_undo; }
list_add(&block->right->tmp_link, &dfs);list_add(&block->left->tmp_link, &dfs);
list_add(&block->right->tmp_link, dfs);list_add(&block->left->tmp_link, dfs);
- list_splice_tail(&allocated, blocks);
Maybe keep this. That way it will be consistent with what drm_buddy_alloc_blocks is doing? Otherwise if there is a good reason for dropping this, then we should also make drm_buddy_alloc_blocks do the same thing. At the moment if chaining allocations with the same list, this will also nuke the existing nodes if we ever hit err_free, but the other path leaves the existing nodes untouched, AFAIK.
changes are added to v10
return 0;
err_undo: @@ -452,14 +479,149 @@ int drm_buddy_alloc_range(struct drm_buddy *mm, buddy = get_buddy(block); if (buddy && (drm_buddy_block_is_free(block) &&
drm_buddy_block_is_free(buddy)))
drm_buddy_block_is_free(buddy))) {list_del(&block->link);- }
+err_free:
- drm_buddy_free_list(mm, blocks);
- return err;
+}
+/**
- __drm_buddy_alloc_range - actual range allocation
- @mm: DRM buddy manager to allocate from
- @start: start of the allowed range for this block
- @size: size of the allocation
- @blocks: output list head to add allocated blocks
- Intended for pre-allocating portions of the address space, for example to
- reserve a block for the initial framebuffer or similar
- Note that it's safe to chain together multiple alloc_ranges
- with the same blocks list
- Returns:
- 0 on success, error code on failure.
- */
+static int __drm_buddy_alloc_range(struct drm_buddy *mm,
u64 start,u64 size,struct list_head *blocks)+{
- LIST_HEAD(dfs);
- int i;
- for (i = 0; i < mm->n_roots; ++i)
list_add_tail(&mm->roots[i]->tmp_link, &dfs);- return __alloc_range(mm, &dfs, start, size, blocks);
+}
+/**
- drm_buddy_alloc_blocks - allocate power-of-two blocks
- @mm: DRM buddy manager to allocate from
- @start: start of the allowed range for this block
- @end: end of the allowed range for this block
- @size: size of the allocation
- @min_page_size: alignment of the allocation
- @blocks: output list head to add allocated blocks
- @flags: DRM_BUDDY_*_ALLOCATION flags
- alloc_range_bias() called on range limitations, which traverses
- the tree and returns the desired block.
- alloc_from_freelist() called when *no* range restrictions
- are enforced, which picks the block from the freelist.
- blocks are allocated in order, the order value here translates to:
- 0 = 2^0 * mm->chunk_size
- 1 = 2^1 * mm->chunk_size
- 2 = 2^2 * mm->chunk_size
- Returns:
- 0 on success, error code on failure.
- */
+int drm_buddy_alloc_blocks(struct drm_buddy *mm,
u64 start, u64 end, u64 size,u64 min_page_size,struct list_head *blocks,unsigned long flags)+{
struct drm_buddy_block *block = NULL;
unsigned int min_order, order;
unsigned long pages;
LIST_HEAD(allocated);
int err;
if (size < mm->chunk_size)
return -EINVAL;if (min_page_size < mm->chunk_size)
return -EINVAL;if (!is_power_of_2(min_page_size))
return -EINVAL;if (!IS_ALIGNED(start | end | size, mm->chunk_size))
return -EINVAL;if (check_range_overflow(start, end, size, mm->size))
return -EINVAL;/* Actual range allocation */
if (start + size == end)
return __drm_buddy_alloc_range(mm, start, size, blocks);pages = size >> ilog2(mm->chunk_size);
order = fls(pages) - 1;
min_order = ilog2(min_page_size) - ilog2(mm->chunk_size);
do {
order = min(order, (unsigned int)fls(pages) - 1);BUG_ON(order > mm->max_order);BUG_ON(order < min_order);do {if (flags & DRM_BUDDY_RANGE_ALLOCATION)/* Allocate traversing within the range */block = alloc_range_bias(mm, start, end, order);else/* Allocate from freelist */block = alloc_from_freelist(mm, order, flags);if (!IS_ERR(block))break;if (order-- == min_order) {err = -ENOSPC;goto err_free;}} while (1);mark_allocated(block);mm->avail -= drm_buddy_block_size(mm, block);kmemleak_update_trace(block);list_add_tail(&block->link, &allocated);pages -= BIT(order);if (!pages)break;} while (1);
list_splice_tail(&allocated, blocks);
return 0;
err_free: drm_buddy_free_list(mm, &allocated); return err; }
-EXPORT_SYMBOL(drm_buddy_alloc_range); +EXPORT_SYMBOL(drm_buddy_alloc_blocks);
/**
- drm_buddy_block_print - print block information
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 6ba314f9836a..1411f4cf1f21 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -36,13 +36,14 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct i915_ttm_buddy_resource *bman_res; struct drm_buddy *mm = &bman->mm;
- unsigned long n_pages;
- unsigned int min_order;
- unsigned long n_pages, lpfn; u64 min_page_size; u64 size; int err;
- GEM_BUG_ON(place->fpfn || place->lpfn);
lpfn = place->lpfn;
if (!lpfn)
lpfn = man->size;bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL); if (!bman_res)
@@ -52,6 +53,9 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, INIT_LIST_HEAD(&bman_res->blocks); bman_res->mm = mm;
- if (place->fpfn || lpfn != man->size)
bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;- GEM_BUG_ON(!bman_res->base.num_pages); size = bman_res->base.num_pages << PAGE_SHIFT;
@@ -60,10 +64,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, min_page_size = bo->page_alignment << PAGE_SHIFT;
GEM_BUG_ON(min_page_size < mm->chunk_size);
- min_order = ilog2(min_page_size) - ilog2(mm->chunk_size);
- if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
unsigned long pages;- size = roundup_pow_of_two(size);
min_order = ilog2(size) - ilog2(mm->chunk_size);
min_page_size = size;pages = size >> ilog2(mm->chunk_size);if (pages > lpfn)lpfn = pages;}
if (size > mm->size) {
@@ -73,34 +83,16 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
n_pages = size >> ilog2(mm->chunk_size);
- do {
struct drm_buddy_block *block;unsigned int order;order = fls(n_pages) - 1;GEM_BUG_ON(order > mm->max_order);GEM_BUG_ON(order < min_order);do {mutex_lock(&bman->lock);block = drm_buddy_alloc_blocks(mm, order);mutex_unlock(&bman->lock);if (!IS_ERR(block))break;if (order-- == min_order) {err = -ENOSPC;goto err_free_blocks;}} while (1);n_pages -= BIT(order);list_add_tail(&block->link, &bman_res->blocks);if (!n_pages)break;- } while (1);
mutex_lock(&bman->lock);
err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
(u64)lpfn << PAGE_SHIFT,(u64)n_pages << PAGE_SHIFT,min_page_size,&bman_res->blocks,bman_res->flags);mutex_unlock(&bman->lock);
if (unlikely(err))
goto err_free_blocks;*res = &bman_res->base; return 0;
@@ -266,10 +258,17 @@ int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct drm_buddy *mm = &bman->mm;
unsigned long flags = 0; int ret;
flags |= DRM_BUDDY_RANGE_ALLOCATION;
mutex_lock(&bman->lock);
- ret = drm_buddy_alloc_range(mm, &bman->reserved, start, size);
ret = drm_buddy_alloc_blocks(mm, start,
start + size,size, mm->chunk_size,&bman->reserved,flags);mutex_unlock(&bman->lock);
return ret;
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h index 312077941411..72c90b432e87 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h @@ -20,6 +20,7 @@ struct drm_buddy;
- @base: struct ttm_resource base class we extend
- @blocks: the list of struct i915_buddy_block for this resource/allocation
- @flags: DRM_BUDDY_*_ALLOCATION flags
- @mm: the struct i915_buddy_mm for this resource
- Extends the struct ttm_resource to manage an address space allocation with
@@ -28,6 +29,7 @@ struct drm_buddy; struct i915_ttm_buddy_resource { struct ttm_resource base; struct list_head blocks;
- unsigned long flags; struct drm_buddy *mm; };
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index f524db152413..865664b90a8a 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -13,15 +13,22 @@
#include <drm/drm_print.h>
-#define range_overflows(start, size, max) ({ \ +#define check_range_overflow(start, end, size, max) ({ \ typeof(start) start__ = (start); \
- typeof(end) end__ = (end);\ typeof(size) size__ = (size); \ typeof(max) max__ = (max); \ (void)(&start__ == &size__); \ (void)(&start__ == &max__); \
- start__ >= max__ || size__ > max__ - start__; \
- (void)(&start__ == &end__); \
- (void)(&end__ == &size__); \
- (void)(&end__ == &max__); \
- start__ >= max__ || end__ > max__ || \
- size__ > end__ - start__; \
Hmm, this doesn't say anything about start >= end. IIUC this will incorrectly still allow something like:
check_range_overflow(PAGE_SIZE, 0ULL, PAGE_SIZE, mm->size)
Perhaps just keep the macro as it was, and then maybe just add something like:
if (end > mm->size) return -EINVAL;
if (range_overflows(start, size, end)) return -EINVAL;
IMO that looks a little easier to understand?
Otherwise I think looks good.
changes are added to v10
})
+#define DRM_BUDDY_RANGE_ALLOCATION (1 << 0)
- struct drm_buddy_block { #define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) #define DRM_BUDDY_HEADER_STATE GENMASK_ULL(11, 10)
@@ -131,12 +138,11 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size);
void drm_buddy_fini(struct drm_buddy *mm);
-struct drm_buddy_block * -drm_buddy_alloc_blocks(struct drm_buddy *mm, unsigned int order);
-int drm_buddy_alloc_range(struct drm_buddy *mm,
struct list_head *blocks,u64 start, u64 size);+int drm_buddy_alloc_blocks(struct drm_buddy *mm,
u64 start, u64 end, u64 size,u64 min_page_size,struct list_head *blocks,unsigned long flags);void drm_buddy_free_block(struct drm_buddy *mm, struct drm_buddy_block *block);
base-commit: e3a3896bf9a189f3992db9fd8622893cf90652cb
dri-devel@lists.freedesktop.org
-
Arunpravin -
Arunpravin
-
Matthew Auld