From: Daniel Vetter daniel.vetter@ffwll.ch

Only ever assigned, never used.

Signed-off-by: Daniel Vetter daniel.vetter@ffwll.ch [glisse: I will re-add if needed for range-restricted allocations] Signed-off-by: Chris Wilson chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_gem.c | 4 +--- drivers/gpu/drm/ttm/ttm_bo.c | 6 ------ drivers/gpu/drm/ttm/ttm_bo_util.c | 2 -- include/drm/drm_mm.h | 1 - 4 files changed, 1 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index a6fbe3b..bfc7269 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -2643,10 +2643,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) if (free_space != NULL) { obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, alignment); - if (obj_priv->gtt_space != NULL) { - obj_priv->gtt_space->private = obj; + if (obj_priv->gtt_space != NULL) obj_priv->gtt_offset = obj_priv->gtt_space->start; - } } if (obj_priv->gtt_space == NULL) { /* If the gtt is empty and we're still having trouble diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 555ebb1..9763288 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -476,7 +476,6 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all) ++put_count; } if (bo->mem.mm_node) { - bo->mem.mm_node->private = NULL; drm_mm_put_block(bo->mem.mm_node); bo->mem.mm_node = NULL; } @@ -670,7 +669,6 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); spin_lock(&glob->lru_lock); if (evict_mem.mm_node) { - evict_mem.mm_node->private = NULL; drm_mm_put_block(evict_mem.mm_node); evict_mem.mm_node = NULL; } @@ -929,8 +927,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, mem->mm_node = node; mem->mem_type = mem_type; mem->placement = cur_flags; - if (node) - node->private = bo; return 0; }

@@ -973,7 +969,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, interruptible, no_wait_reserve, no_wait_gpu); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; - mem->mm_node->private = bo; return 0; } if (ret == -ERESTARTSYS) @@ -1029,7 +1024,6 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, out_unlock: if (ret && mem.mm_node) { spin_lock(&glob->lru_lock); - mem.mm_node->private = NULL; drm_mm_put_block(mem.mm_node); spin_unlock(&glob->lru_lock); } diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 13012a1..7cffb3e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -353,8 +353,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->vm_node = NULL;

fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); - if (fbo->mem.mm_node) - fbo->mem.mm_node->private = (void *)fbo; kref_init(&fbo->list_kref); kref_init(&fbo->kref); fbo->destroy = &ttm_transfered_destroy; diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h index 4c10be3..da94071 100644 --- a/include/drm/drm_mm.h +++ b/include/drm/drm_mm.h @@ -48,7 +48,6 @@ struct drm_mm_node { unsigned long start; unsigned long size; struct drm_mm *mm; - void *private; };

struct drm_mm {

From: Daniel Vetter daniel.vetter@ffwll.ch

Yeah, I've kinda noticed that fl_entry is the free stack. Still give it (and the memory node list ml_entry) decent names.

Signed-off-by: Daniel Vetter daniel.vetter@ffwll.ch Acked-by: Thomas Hellstrom thellstrom@vmwgfx.com Signed-off-by: Chris Wilson chris@chris-wilson.co.uk --- drivers/gpu/drm/drm_mm.c | 72 ++++++++++++++++++++++----------------------- include/drm/drm_mm.h | 11 ++++-- 2 files changed, 42 insertions(+), 41 deletions(-)

diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index a5a7a16..d2267ff 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -64,8 +64,8 @@ static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic) else { child = list_entry(mm->unused_nodes.next, - struct drm_mm_node, fl_entry); - list_del(&child->fl_entry); + struct drm_mm_node, free_stack); + list_del(&child->free_stack); --mm->num_unused; } spin_unlock(&mm->unused_lock); @@ -94,7 +94,7 @@ int drm_mm_pre_get(struct drm_mm *mm) return ret; } ++mm->num_unused; - list_add_tail(&node->fl_entry, &mm->unused_nodes); + list_add_tail(&node->free_stack, &mm->unused_nodes); } spin_unlock(&mm->unused_lock); return 0; @@ -116,8 +116,8 @@ static int drm_mm_create_tail_node(struct drm_mm *mm, child->start = start; child->mm = mm;

- list_add_tail(&child->ml_entry, &mm->ml_entry); - list_add_tail(&child->fl_entry, &mm->fl_entry); + list_add_tail(&child->node_list, &mm->node_list); + list_add_tail(&child->free_stack, &mm->free_stack);

return 0; } @@ -132,15 +132,15 @@ static struct drm_mm_node *drm_mm_split_at_start(struct drm_mm_node *parent, if (unlikely(child == NULL)) return NULL;

- INIT_LIST_HEAD(&child->fl_entry); + INIT_LIST_HEAD(&child->free_stack);

child->free = 0; child->size = size; child->start = parent->start; child->mm = parent->mm;

- list_add_tail(&child->ml_entry, &parent->ml_entry); - INIT_LIST_HEAD(&child->fl_entry); + list_add_tail(&child->node_list, &parent->node_list); + INIT_LIST_HEAD(&child->free_stack);

parent->size -= size; parent->start += size; @@ -168,7 +168,7 @@ struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *node, }

if (node->size == size) { - list_del_init(&node->fl_entry); + list_del_init(&node->free_stack); node->free = 0; } else { node = drm_mm_split_at_start(node, size, atomic); @@ -206,7 +206,7 @@ struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *node, }

if (node->size == size) { - list_del_init(&node->fl_entry); + list_del_init(&node->free_stack); node->free = 0; } else { node = drm_mm_split_at_start(node, size, atomic); @@ -228,8 +228,8 @@ void drm_mm_put_block(struct drm_mm_node *cur) {

struct drm_mm *mm = cur->mm; - struct list_head *cur_head = &cur->ml_entry; - struct list_head *root_head = &mm->ml_entry; + struct list_head *cur_head = &cur->node_list; + struct list_head *root_head = &mm->node_list; struct drm_mm_node *prev_node = NULL; struct drm_mm_node *next_node;

@@ -237,7 +237,7 @@ void drm_mm_put_block(struct drm_mm_node *cur)

if (cur_head->prev != root_head) { prev_node = - list_entry(cur_head->prev, struct drm_mm_node, ml_entry); + list_entry(cur_head->prev, struct drm_mm_node, node_list); if (prev_node->free) { prev_node->size += cur->size; merged = 1; @@ -245,15 +245,15 @@ void drm_mm_put_block(struct drm_mm_node *cur) } if (cur_head->next != root_head) { next_node = - list_entry(cur_head->next, struct drm_mm_node, ml_entry); + list_entry(cur_head->next, struct drm_mm_node, node_list); if (next_node->free) { if (merged) { prev_node->size += next_node->size; - list_del(&next_node->ml_entry); - list_del(&next_node->fl_entry); + list_del(&next_node->node_list); + list_del(&next_node->free_stack); spin_lock(&mm->unused_lock); if (mm->num_unused < MM_UNUSED_TARGET) { - list_add(&next_node->fl_entry, + list_add(&next_node->free_stack, &mm->unused_nodes); ++mm->num_unused; } else @@ -268,12 +268,12 @@ void drm_mm_put_block(struct drm_mm_node *cur) } if (!merged) { cur->free = 1; - list_add(&cur->fl_entry, &mm->fl_entry); + list_add(&cur->free_stack, &mm->free_stack); } else { - list_del(&cur->ml_entry); + list_del(&cur->node_list); spin_lock(&mm->unused_lock); if (mm->num_unused < MM_UNUSED_TARGET) { - list_add(&cur->fl_entry, &mm->unused_nodes); + list_add(&cur->free_stack, &mm->unused_nodes); ++mm->num_unused; } else kfree(cur); @@ -287,7 +287,6 @@ struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm, unsigned long size, unsigned alignment, int best_match) { - const struct list_head *free_stack = &mm->fl_entry; struct drm_mm_node *entry; struct drm_mm_node *best; unsigned long best_size; @@ -296,7 +295,7 @@ struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm, best = NULL; best_size = ~0UL;

- list_for_each_entry(entry, free_stack, fl_entry) { + list_for_each_entry(entry, &mm->free_stack, free_stack) { wasted = 0;

if (entry->size < size) @@ -329,7 +328,6 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm, unsigned long end, int best_match) { - const struct list_head *free_stack = &mm->fl_entry; struct drm_mm_node *entry; struct drm_mm_node *best; unsigned long best_size; @@ -338,7 +336,7 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm, best = NULL; best_size = ~0UL;

- list_for_each_entry(entry, free_stack, fl_entry) { + list_for_each_entry(entry, &mm->free_stack, free_stack) { wasted = 0;

if (entry->size < size) @@ -373,7 +371,7 @@ EXPORT_SYMBOL(drm_mm_search_free_in_range);

int drm_mm_clean(struct drm_mm * mm) { - struct list_head *head = &mm->ml_entry; + struct list_head *head = &mm->node_list;

return (head->next->next == head); } @@ -381,8 +379,8 @@ EXPORT_SYMBOL(drm_mm_clean);

int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size) { - INIT_LIST_HEAD(&mm->ml_entry); - INIT_LIST_HEAD(&mm->fl_entry); + INIT_LIST_HEAD(&mm->node_list); + INIT_LIST_HEAD(&mm->free_stack); INIT_LIST_HEAD(&mm->unused_nodes); mm->num_unused = 0; spin_lock_init(&mm->unused_lock); @@ -393,25 +391,25 @@ EXPORT_SYMBOL(drm_mm_init);

void drm_mm_takedown(struct drm_mm * mm) { - struct list_head *bnode = mm->fl_entry.next; + struct list_head *bnode = mm->free_stack.next; struct drm_mm_node *entry; struct drm_mm_node *next;

- entry = list_entry(bnode, struct drm_mm_node, fl_entry); + entry = list_entry(bnode, struct drm_mm_node, free_stack);

- if (entry->ml_entry.next != &mm->ml_entry || - entry->fl_entry.next != &mm->fl_entry) { + if (entry->node_list.next != &mm->node_list || + entry->free_stack.next != &mm->free_stack) { DRM_ERROR("Memory manager not clean. Delaying takedown\n"); return; }

- list_del(&entry->fl_entry); - list_del(&entry->ml_entry); + list_del(&entry->free_stack); + list_del(&entry->node_list); kfree(entry);

spin_lock(&mm->unused_lock); - list_for_each_entry_safe(entry, next, &mm->unused_nodes, fl_entry) { - list_del(&entry->fl_entry); + list_for_each_entry_safe(entry, next, &mm->unused_nodes, free_stack) { + list_del(&entry->free_stack); kfree(entry); --mm->num_unused; } @@ -426,7 +424,7 @@ void drm_mm_debug_table(struct drm_mm *mm, const char *prefix) struct drm_mm_node *entry; int total_used = 0, total_free = 0, total = 0;

- list_for_each_entry(entry, &mm->ml_entry, ml_entry) { + list_for_each_entry(entry, &mm->node_list, node_list) { printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8ld: %s\n", prefix, entry->start, entry->start + entry->size, entry->size, entry->free ? "free" : "used"); @@ -447,7 +445,7 @@ int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm) struct drm_mm_node *entry; int total_used = 0, total_free = 0, total = 0;

- list_for_each_entry(entry, &mm->ml_entry, ml_entry) { + list_for_each_entry(entry, &mm->node_list, node_list) { seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: %s\n", entry->start, entry->start + entry->size, entry->size, entry->free ? "free" : "used"); total += entry->size; if (entry->free) diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h index da94071..e8740cc 100644 --- a/include/drm/drm_mm.h +++ b/include/drm/drm_mm.h @@ -42,8 +42,8 @@ #endif

struct drm_mm_node { - struct list_head fl_entry; - struct list_head ml_entry; + struct list_head free_stack; + struct list_head node_list; int free; unsigned long start; unsigned long size; @@ -51,8 +51,11 @@ struct drm_mm_node { };

struct drm_mm { - struct list_head fl_entry; - struct list_head ml_entry; + /* List of free memory blocks, most recently freed ordered. */ + struct list_head free_stack; + /* List of all memory nodes, ordered according to the (increasing) start + * address of the memory node. */ + struct list_head node_list; struct list_head unused_nodes; int num_unused; spinlock_t unused_lock;

From: Daniel Vetter daniel.vetter@ffwll.ch

These helper functions can be used to efficiently scan lru list for eviction. Eviction becomes a three stage process: 1. Scanning through the lru list until a suitable hole has been found. 2. Scan backwards to restore drm_mm consistency and find out which objects fall into the hole. 3. Evict the objects that fall into the hole.

These helper functions don't allocate any memory (at the price of not allowing any other concurrent operations). Hence this can also be used for ttm (which does lru scanning under a spinlock).

Evicting objects in this fashion should be more fair than the current approach by i915 (scan the lru for a object large enough to contain the new object). It's also more efficient than the current approach used by ttm (uncoditionally evict objects from the lru until there's enough free space).

Signed-Off-by: Daniel Vetter daniel.vetter@ffwll.ch Acked-by: Thomas Hellstrom thellstrom@vmwgfx.com Signed-off-by: Chris Wilson chris@chris-wilson.co.uk --- drivers/gpu/drm/drm_mm.c | 167 ++++++++++++++++++++++++++++++++++++++++++++- include/drm/drm_mm.h | 15 ++++- 2 files changed, 177 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index fd86a6c..da99edc 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -53,9 +53,9 @@ static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic) struct drm_mm_node *child;

if (atomic) - child = kmalloc(sizeof(*child), GFP_ATOMIC); + child = kzalloc(sizeof(*child), GFP_ATOMIC); else - child = kmalloc(sizeof(*child), GFP_KERNEL); + child = kzalloc(sizeof(*child), GFP_KERNEL);

if (unlikely(child == NULL)) { spin_lock(&mm->unused_lock); @@ -85,7 +85,7 @@ int drm_mm_pre_get(struct drm_mm *mm) spin_lock(&mm->unused_lock); while (mm->num_unused < MM_UNUSED_TARGET) { spin_unlock(&mm->unused_lock); - node = kmalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc(sizeof(*node), GFP_KERNEL); spin_lock(&mm->unused_lock);

if (unlikely(node == NULL)) { @@ -134,7 +134,6 @@ static struct drm_mm_node *drm_mm_split_at_start(struct drm_mm_node *parent,

INIT_LIST_HEAD(&child->free_stack);

- child->free = 0; child->size = size; child->start = parent->start; child->mm = parent->mm; @@ -235,6 +234,9 @@ void drm_mm_put_block(struct drm_mm_node *cur)

int merged = 0;

+ BUG_ON(cur->scanned_block || cur->scanned_prev_free + || cur->scanned_next_free); + if (cur_head->prev != root_head) { prev_node = list_entry(cur_head->prev, struct drm_mm_node, node_list); @@ -312,6 +314,8 @@ struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm, struct drm_mm_node *best; unsigned long best_size;

+ BUG_ON(mm->scanned_blocks); + best = NULL; best_size = ~0UL;

@@ -343,6 +347,8 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm, struct drm_mm_node *best; unsigned long best_size;

+ BUG_ON(mm->scanned_blocks); + best = NULL; best_size = ~0UL;

@@ -366,6 +372,158 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm, } EXPORT_SYMBOL(drm_mm_search_free_in_range);

+/** + * Initializa lru scanning. + * + * This simply sets up the scanning routines with the parameters for the desired + * hole. + * + * Warning: As long as the scan list is non-empty, no other operations than + * adding/removing nodes to/from the scan list are allowed. + */ +void drm_mm_init_scan(struct drm_mm *mm, unsigned long size, + unsigned alignment) +{ + mm->scan_alignment = alignment; + mm->scan_size = size; + mm->scanned_blocks = 0; + mm->scan_hit_start = 0; + mm->scan_hit_size = 0; +} +EXPORT_SYMBOL(drm_mm_init_scan); + +/** + * Add a node to the scan list that might be freed to make space for the desired + * hole. + * + * Returns non-zero, if a hole has been found, zero otherwise. + */ +int drm_mm_scan_add_block(struct drm_mm_node *node) +{ + struct drm_mm *mm = node->mm; + struct list_head *prev_free, *next_free; + struct drm_mm_node *prev_node, *next_node; + + mm->scanned_blocks++; + + prev_free = next_free = NULL; + + BUG_ON(node->free); + node->scanned_block = 1; + node->free = 1; + + if (node->node_list.prev != &mm->node_list) { + prev_node = list_entry(node->node_list.prev, struct drm_mm_node, + node_list); + + if (prev_node->free) { + list_del(&prev_node->node_list); + + node->start = prev_node->start; + node->size += prev_node->size; + + prev_node->scanned_prev_free = 1; + + prev_free = &prev_node->free_stack; + } + } + + if (node->node_list.next != &mm->node_list) { + next_node = list_entry(node->node_list.next, struct drm_mm_node, + node_list); + + if (next_node->free) { + list_del(&next_node->node_list); + + node->size += next_node->size; + + next_node->scanned_next_free = 1; + + next_free = &next_node->free_stack; + } + } + + /* The free_stack list is not used for allocated objects, so these two + * pointers can be abused (as long as no allocations in this memory + * manager happens). */ + node->free_stack.prev = prev_free; + node->free_stack.next = next_free; + + if (check_free_mm_node(node, mm->scan_size, mm->scan_alignment)) { + mm->scan_hit_start = node->start; + mm->scan_hit_size = node->size; + + return 1; + } + + return 0; +} +EXPORT_SYMBOL(drm_mm_scan_add_block); + +/** + * Remove a node from the scan list. + * + * Nodes _must_ be removed in the exact same order from the scan list as they + * have been added, otherwise the internal state of the memory manager will be + * corrupted. + * + * When the scan list is empty, the selected memory nodes can be freed. An + * immediatly following drm_mm_search_free with best_match = 0 will then return + * the just freed block (because its at the top of the free_stack list). + * + * Returns one if this block should be evicted, zero otherwise. Will always + * return zero when no hole has been found. + */ +int drm_mm_scan_remove_block(struct drm_mm_node *node) +{ + struct drm_mm *mm = node->mm; + struct drm_mm_node *prev_node, *next_node; + + mm->scanned_blocks--; + + BUG_ON(!node->scanned_block); + node->scanned_block = 0; + node->free = 0; + + prev_node = list_entry(node->free_stack.prev, struct drm_mm_node, + free_stack); + next_node = list_entry(node->free_stack.next, struct drm_mm_node, + free_stack); + + if (prev_node) { + BUG_ON(!prev_node->scanned_prev_free); + prev_node->scanned_prev_free = 0; + + list_add_tail(&prev_node->node_list, &node->node_list); + + node->start = prev_node->start + prev_node->size; + node->size -= prev_node->size; + } + + if (next_node) { + BUG_ON(!next_node->scanned_next_free); + next_node->scanned_next_free = 0; + + list_add(&next_node->node_list, &node->node_list); + + node->size -= next_node->size; + } + + INIT_LIST_HEAD(&node->free_stack); + + /* Only need to check for containement because start&size for the + * complete resulting free block (not just the desired part) is + * stored. */ + if (node->start >= mm->scan_hit_start && + node->start + node->size + <= mm->scan_hit_start + mm->scan_hit_size) { + return 1; + } + + return 0; +} +EXPORT_SYMBOL(drm_mm_scan_remove_block); + int drm_mm_clean(struct drm_mm * mm) { struct list_head *head = &mm->node_list; @@ -380,6 +538,7 @@ int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size) INIT_LIST_HEAD(&mm->free_stack); INIT_LIST_HEAD(&mm->unused_nodes); mm->num_unused = 0; + mm->scanned_blocks = 0; spin_lock_init(&mm->unused_lock);

return drm_mm_create_tail_node(mm, start, size, 0); diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h index e8740cc..bf01531 100644 --- a/include/drm/drm_mm.h +++ b/include/drm/drm_mm.h @@ -44,7 +44,10 @@ struct drm_mm_node { struct list_head free_stack; struct list_head node_list; - int free; + unsigned free : 1; + unsigned scanned_block : 1; + unsigned scanned_prev_free : 1; + unsigned scanned_next_free : 1; unsigned long start; unsigned long size; struct drm_mm *mm; @@ -59,6 +62,11 @@ struct drm_mm { struct list_head unused_nodes; int num_unused; spinlock_t unused_lock; + unsigned scan_alignment; + unsigned long scan_size; + unsigned long scan_hit_start; + unsigned scan_hit_size; + unsigned scanned_blocks; };

/* @@ -135,6 +143,11 @@ static inline struct drm_mm *drm_get_mm(struct drm_mm_node *block) return block->mm; }

+void drm_mm_init_scan(struct drm_mm *mm, unsigned long size, + unsigned alignment); +int drm_mm_scan_add_block(struct drm_mm_node *node); +int drm_mm_scan_remove_block(struct drm_mm_node *node); + extern void drm_mm_debug_table(struct drm_mm *mm, const char *prefix); #ifdef CONFIG_DEBUG_FS int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm);

This will be used by the eviction logic to maintain fairness between the rings.

Signed-off-by: Chris Wilson chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_drv.h | 3 +- drivers/gpu/drm/i915/i915_gem.c | 2 + drivers/gpu/drm/i915/intel_ringbuffer.c | 46 +++++++++++++++++------------- drivers/gpu/drm/i915/intel_ringbuffer.h | 1 - 4 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 98e6980..65cf336 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -236,6 +236,7 @@ typedef struct drm_i915_private { struct pci_dev *bridge_dev; struct intel_ring_buffer render_ring; struct intel_ring_buffer bsd_ring; + uint32_t next_seqno;

drm_dma_handle_t *status_page_dmah; void *seqno_page; @@ -553,8 +554,6 @@ typedef struct drm_i915_private { */ struct delayed_work retire_work;

- uint32_t next_gem_seqno; - /** * Waiting sequence number, if any */ diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index d4bcf71..1b867c4 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -4701,6 +4701,8 @@ i915_gem_init_ringbuffer(struct drm_device *dev) goto cleanup_render_ring; }

+ dev_priv->next_seqno = 1; + return 0;

cleanup_render_ring: diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 2a3d2fa..ec04238 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -33,18 +33,35 @@ #include "i915_drm.h" #include "i915_trace.h"

+static u32 i915_gem_get_seqno(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + u32 seqno; + + seqno = dev_priv->next_seqno; + + /* reserve 0 for non-seqno */ + if (++dev_priv->next_seqno == 0) + dev_priv->next_seqno = 1; + + return seqno; +} + static void render_ring_flush(struct drm_device *dev, struct intel_ring_buffer *ring, u32 invalidate_domains, u32 flush_domains) { + drm_i915_private_t *dev_priv = dev->dev_private; + u32 cmd; + #if WATCH_EXEC DRM_INFO("%s: invalidate %08x flush %08x\n", __func__, invalidate_domains, flush_domains); #endif - u32 cmd; - trace_i915_gem_request_flush(dev, ring->next_seqno, + + trace_i915_gem_request_flush(dev, dev_priv->next_seqno, invalidate_domains, flush_domains);

if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) { @@ -233,9 +250,10 @@ render_ring_add_request(struct drm_device *dev, struct drm_file *file_priv, u32 flush_domains) { - u32 seqno; drm_i915_private_t *dev_priv = dev->dev_private; - seqno = intel_ring_get_seqno(dev, ring); + u32 seqno; + + seqno = i915_gem_get_seqno(dev);

if (IS_GEN6(dev)) { BEGIN_LP_RING(6); @@ -405,7 +423,9 @@ bsd_ring_add_request(struct drm_device *dev, u32 flush_domains) { u32 seqno; - seqno = intel_ring_get_seqno(dev, ring); + + seqno = i915_gem_get_seqno(dev); + intel_ring_begin(dev, ring, 4); intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX); intel_ring_emit(dev, ring, @@ -539,7 +559,7 @@ render_ring_dispatch_gem_execbuffer(struct drm_device *dev, exec_start = (uint32_t) exec_offset + exec->batch_start_offset; exec_len = (uint32_t) exec->batch_len;

- trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1); + trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);

pipeline = exec->flags & I915_EXEC_PIPELINE_MASK; if (pipeline != ring->pipeline) { @@ -838,18 +858,6 @@ void intel_fill_struct(struct drm_device *dev, intel_ring_advance(dev, ring); }

-u32 intel_ring_get_seqno(struct drm_device *dev, - struct intel_ring_buffer *ring) -{ - u32 seqno; - seqno = ring->next_seqno; - - /* reserve 0 for non-seqno */ - if (++ring->next_seqno == 0) - ring->next_seqno = 1; - return seqno; -} - struct intel_ring_buffer render_ring = { .name = "render ring", .regs = { @@ -867,7 +875,6 @@ struct intel_ring_buffer render_ring = { .head = 0, .tail = 0, .space = 0, - .next_seqno = 1, .user_irq_refcount = 0, .irq_gem_seqno = 0, .waiting_gem_seqno = 0, @@ -906,7 +913,6 @@ struct intel_ring_buffer bsd_ring = { .head = 0, .tail = 0, .space = 0, - .next_seqno = 1, .user_irq_refcount = 0, .irq_gem_seqno = 0, .waiting_gem_seqno = 0, diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index ff5db57..1c50ae1 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -26,7 +26,6 @@ struct intel_ring_buffer { unsigned int head; unsigned int tail; unsigned int space; - u32 next_seqno; struct intel_hw_status_page status_page;

u32 irq_gem_seqno; /* last seq seem at irq time */

Based in a large part upon Daniel Vetter's implementation and adapted for handling multiple rings in a single pass.

This should lead to better gtt usage and fixes the page-fault-of-doom triggered. The fairness is provided by scanning through the GTT space amalgamating space in rendering order. As soon as we have a contiguous space in the GTT large enough for the new object (and its alignment), evict any object which lies within that space. This should keep more objects resident in the GTT.

Doing throughput testing on a PineView machine with cairo-perf-trace indicates that there is very little difference with the new LRU scan, perhaps a small improvement.

Reference:

Bug 15911 - Intermittent X crash (freeze) https://bugzilla.kernel.org/show_bug.cgi?id=15911

Bug 20152 - cannot view JPG in firefox when running UXA https://bugs.freedesktop.org/show_bug.cgi?id=20152

Bug 24369 - Hang when scrolling firefox page with window in front https://bugs.freedesktop.org/show_bug.cgi?id=24369

Bug 28478 - Intermittent graphics lockups due to overflow/loop https://bugs.freedesktop.org/show_bug.cgi?id=28478

Signed-off-by: Chris Wilson chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_drv.h | 2 + drivers/gpu/drm/i915/i915_gem_evict.c | 259 +++++++++++++++------------------ 2 files changed, 121 insertions(+), 140 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index b6a0bd0..8e9e6a9 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -652,6 +652,8 @@ struct drm_i915_gem_object { struct list_head list; /** This object's place on GPU write list */ struct list_head gpu_write_list; + /** This object's place on eviction list */ + struct list_head evict_list;

/** * This is set if the object is on the active or flushing lists diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index bf4d199..87efbab 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -31,170 +31,149 @@ #include "i915_drv.h" #include "i915_drm.h"

-static inline int -i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv) -{ - return obj_priv->madv == I915_MADV_DONTNEED; -} - -static int -i915_gem_scan_inactive_list_and_evict(struct drm_device *dev, int min_size, - unsigned alignment, int *found) +static struct drm_i915_gem_object * +i915_gem_next_active_object(struct drm_device *dev, + struct list_head **render_iter, + struct list_head **bsd_iter) { drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; - struct drm_gem_object *best = NULL; - struct drm_gem_object *first = NULL; + struct drm_i915_gem_object *render_obj = NULL, *bsd_obj = NULL;

- /* Try to find the smallest clean object */ - list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) { - struct drm_gem_object *obj = &obj_priv->base; - if (obj->size >= min_size) { - if ((!obj_priv->dirty || - i915_gem_object_is_purgeable(obj_priv)) && - (!best || obj->size < best->size)) { - best = obj; - if (best->size == min_size) - break; - } - if (!first) - first = obj; - } - } - - obj = best ? best : first; - - if (!obj) { - *found = 0; - return 0; - } + if (*render_iter != &dev_priv->render_ring.active_list) + render_obj = list_entry(*render_iter, + struct drm_i915_gem_object, + list);

- *found = 1; + if (HAS_BSD(dev)) { + if (*bsd_iter != &dev_priv->bsd_ring.active_list) + bsd_obj = list_entry(*bsd_iter, + struct drm_i915_gem_object, + list);

-#if WATCH_LRU - DRM_INFO("%s: evicting %p\n", __func__, obj); -#endif - obj_priv = to_intel_bo(obj); - BUG_ON(obj_priv->pin_count != 0); - BUG_ON(obj_priv->active); + if (render_obj == NULL) { + *bsd_iter = (*bsd_iter)->next; + return bsd_obj; + }

- /* Wait on the rendering and unbind the buffer. */ - return i915_gem_object_unbind(obj); -} + if (bsd_obj == NULL) { + *render_iter = (*render_iter)->next; + return render_obj; + }

-static void -i915_gem_flush_ring(struct drm_device *dev, - uint32_t invalidate_domains, - uint32_t flush_domains, - struct intel_ring_buffer *ring) -{ - if (flush_domains & I915_GEM_DOMAIN_CPU) - drm_agp_chipset_flush(dev); - ring->flush(dev, ring, - invalidate_domains, - flush_domains); + /* XXX can we handle seqno wrapping? */ + if (render_obj->last_rendering_seqno < bsd_obj->last_rendering_seqno) { + *render_iter = (*render_iter)->next; + return render_obj; + } else { + *bsd_iter = (*bsd_iter)->next; + return bsd_obj; + } + } else { + *render_iter = (*render_iter)->next; + return render_obj; + } }

int -i915_gem_evict_something(struct drm_device *dev, - int min_size, unsigned alignment) +i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment) { drm_i915_private_t *dev_priv = dev->dev_private; - int ret, found; - - struct intel_ring_buffer *render_ring = &dev_priv->render_ring; - struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring; - for (;;) { - i915_gem_retire_requests(dev, render_ring); - - if (HAS_BSD(dev)) - i915_gem_retire_requests(dev, bsd_ring); - - /* If there's an inactive buffer available now, grab it - * and be done. - */ - ret = i915_gem_scan_inactive_list_and_evict(dev, min_size, - alignment, - &found); - if (found) - return ret; + struct list_head eviction_list, unwind_list; + struct drm_i915_gem_object *obj_priv, *tmp_obj_priv; + struct list_head *render_iter, *bsd_iter; + int ret = 0; + + i915_gem_retire_requests(dev, &dev_priv->render_ring); + if (HAS_BSD(dev)) + i915_gem_retire_requests(dev, &dev_priv->bsd_ring); + + /* Re-check for free space after retiring requests */ + if (drm_mm_search_free(&dev_priv->mm.gtt_space, + min_size, alignment, 0)) + return 0;

- /* If we didn't get anything, but the ring is still processing - * things, wait for the next to finish and hopefully leave us - * a buffer to evict. - */ - if (!list_empty(&render_ring->request_list)) { - struct drm_i915_gem_request *request;

- request = list_first_entry(&render_ring->request_list, - struct drm_i915_gem_request, - list); + INIT_LIST_HEAD(&unwind_list); + drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);

- ret = i915_do_wait_request(dev, request->seqno, true, request->ring); - if (ret) - return ret; + /* First see if there is a large enough contiguous idle region... */ + list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) { + list_add(&obj_priv->evict_list, &unwind_list); + if (drm_mm_scan_add_block(obj_priv->gtt_space)) + goto found; + }

+ /* Now merge in the soon-to-be-expired objects... */ + render_iter = dev_priv->render_ring.active_list.next; + bsd_iter = dev_priv->bsd_ring.active_list.next; + while ((obj_priv = i915_gem_next_active_object(dev, &render_iter, &bsd_iter)) != NULL) { + /* Does the object require an outstanding flush? */ + if (obj_priv->base.write_domain || obj_priv->pin_count) continue; - }

- if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) { - struct drm_i915_gem_request *request; - - request = list_first_entry(&bsd_ring->request_list, - struct drm_i915_gem_request, - list); + list_add(&obj_priv->evict_list, &unwind_list); + if (drm_mm_scan_add_block(obj_priv->gtt_space)) + goto found; + }

- ret = i915_do_wait_request(dev, request->seqno, true, request->ring); - if (ret) - return ret; + /* Finally add anything with a pending flush (in order of retirement). */ + list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) { + if (obj_priv->pin_count) + continue;

+ list_add(&obj_priv->evict_list, &unwind_list); + if (drm_mm_scan_add_block(obj_priv->gtt_space)) + goto found; + } + render_iter = dev_priv->render_ring.active_list.next; + bsd_iter = dev_priv->bsd_ring.active_list.next; + while ((obj_priv = i915_gem_next_active_object(dev, &render_iter, &bsd_iter)) != NULL) { + if (! obj_priv->base.write_domain || obj_priv->pin_count) continue; - }

- /* If we didn't have anything on the request list but there - * are buffers awaiting a flush, emit one and try again. - * When we wait on it, those buffers waiting for that flush - * will get moved to inactive. - */ - if (!list_empty(&dev_priv->mm.flushing_list)) { - struct drm_gem_object *obj = NULL; - struct drm_i915_gem_object *obj_priv; - - /* Find an object that we can immediately reuse */ - list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) { - obj = &obj_priv->base; - if (obj->size >= min_size) - break; - - obj = NULL; - } - - if (obj != NULL) { - uint32_t seqno; - - i915_gem_flush_ring(dev, - obj->write_domain, - obj->write_domain, - obj_priv->ring); - seqno = i915_add_request(dev, NULL, - obj->write_domain, - obj_priv->ring); - if (seqno == 0) - return -ENOMEM; - continue; - } + list_add(&obj_priv->evict_list, &unwind_list); + if (drm_mm_scan_add_block(obj_priv->gtt_space)) + goto found; + } + + /* Nothing found, clean up and bail out! */ + list_for_each_entry(obj_priv, &unwind_list, evict_list) { + ret = drm_mm_scan_remove_block(obj_priv->gtt_space); + BUG_ON(ret); + } + + return i915_gem_evict_everything(dev); + +found: + INIT_LIST_HEAD(&eviction_list); + list_for_each_entry_safe(obj_priv, tmp_obj_priv, &unwind_list, evict_list) { + if (drm_mm_scan_remove_block(obj_priv->gtt_space)) { + /* drm_mm doesn't allow any other other operations while + * scanning, therefore store to be evicted objects on a + * temporary list. */ + list_move(&obj_priv->evict_list, &eviction_list); } + }

- /* If we didn't do any of the above, there's no single buffer - * large enough to swap out for the new one, so just evict - * everything and start again. (This should be rare.) - */ - if (!list_empty(&dev_priv->mm.inactive_list)) - return i915_gem_evict_inactive(dev); - else - return i915_gem_evict_everything(dev); + /* Unbinding will emit any required flushes */ + list_for_each_entry_safe(obj_priv, tmp_obj_priv, + &eviction_list, evict_list) { +#if WATCH_LRU + DRM_INFO("%s: evicting %p\n", __func__, obj); +#endif + ret = i915_gem_object_unbind(&obj_priv->base); + if (ret) + return ret; } + + /* The just created free hole should be on the top of the free stack + * maintained by drm_mm, so this BUG_ON actually executes in O(1). + * Furthermore all accessed data has just recently been used, so it + * should be really fast, too. */ + BUG_ON(!drm_mm_search_free(&dev_priv->mm.gtt_space, min_size, + alignment, 0)); + + return 0; }

int