From: Jerome Glisse jglisse@redhat.com

We need to sync with the GFX ring as ttm might have schedule bo move on it and new command scheduled for other ring need to wait for bo data to be in place.

Signed-off-by: Jerome Glisse jglisse@redhat.com Reviewed by: Christian König christian.koenig@amd.com --- drivers/gpu/drm/radeon/radeon_cs.c | 12 ++++++------ include/drm/radeon_drm.h | 1 - 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index c66beb1..289b0d7 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -122,15 +122,15 @@ static int radeon_cs_sync_rings(struct radeon_cs_parser *p) int i, r;

for (i = 0; i < p->nrelocs; i++) { + struct radeon_fence *fence; + if (!p->relocs[i].robj || !p->relocs[i].robj->tbo.sync_obj) continue;

- if (!(p->relocs[i].flags & RADEON_RELOC_DONT_SYNC)) { - struct radeon_fence *fence = p->relocs[i].robj->tbo.sync_obj; - if (fence->ring != p->ring && !radeon_fence_signaled(fence)) { - sync_to_ring[fence->ring] = true; - need_sync = true; - } + fence = p->relocs[i].robj->tbo.sync_obj; + if (fence->ring != p->ring && !radeon_fence_signaled(fence)) { + sync_to_ring[fence->ring] = true; + need_sync = true; } }

diff --git a/include/drm/radeon_drm.h b/include/drm/radeon_drm.h index 7c491b4..5805686 100644 --- a/include/drm/radeon_drm.h +++ b/include/drm/radeon_drm.h @@ -926,7 +926,6 @@ struct drm_radeon_cs_chunk { };

/* drm_radeon_cs_reloc.flags */ -#define RADEON_RELOC_DONT_SYNC 0x01

struct drm_radeon_cs_reloc { uint32_t handle;

From: Jerome Glisse jglisse@redhat.com

This convert fence to use uint64_t sequence number intention is to use the fact that uin64_t is big enough that we don't need to care about wrap around.

Tested with and without writeback using 0xFFFFF000 as initial fence sequence and thus allowing to test the wrap around from 32bits to 64bits.

v2: Add comment about possible race btw CPU & GPU, add comment stressing that we need 2 dword aligned for R600_WB_EVENT_OFFSET Read fence sequenc in reverse order of GPU write them so we mitigate the race btw CPU and GPU.

v3: Drop the need for ring to emit the 64bits fence, and just have each ring emit the lower 32bits of the fence sequence. We handle the wrap over 32bits in fence_process.

v4: Just a small optimization: Don't reread the last_seq value if loop restarts, since we already know its value anyway. Also start at zero not one for seq value and use pre instead of post increment in emmit, otherwise wait_empty will deadlock.

Signed-off-by: Jerome Glisse jglisse@redhat.com Signed-off-by: Christian König deathsimple@vodafone.de --- drivers/gpu/drm/radeon/radeon.h | 39 ++++++----- drivers/gpu/drm/radeon/radeon_fence.c | 116 +++++++++++++++++++++++---------- drivers/gpu/drm/radeon/radeon_ring.c | 9 +-- 3 files changed, 107 insertions(+), 57 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index e99ea81..cdf46bc 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -100,28 +100,32 @@ extern int radeon_lockup_timeout; * Copy from radeon_drv.h so we don't have to include both and have conflicting * symbol; */ -#define RADEON_MAX_USEC_TIMEOUT 100000 /* 100 ms */ -#define RADEON_FENCE_JIFFIES_TIMEOUT (HZ / 2) +#define RADEON_MAX_USEC_TIMEOUT 100000 /* 100 ms */ +#define RADEON_FENCE_JIFFIES_TIMEOUT (HZ / 2) /* RADEON_IB_POOL_SIZE must be a power of 2 */ -#define RADEON_IB_POOL_SIZE 16 -#define RADEON_DEBUGFS_MAX_COMPONENTS 32 -#define RADEONFB_CONN_LIMIT 4 -#define RADEON_BIOS_NUM_SCRATCH 8 +#define RADEON_IB_POOL_SIZE 16 +#define RADEON_DEBUGFS_MAX_COMPONENTS 32 +#define RADEONFB_CONN_LIMIT 4 +#define RADEON_BIOS_NUM_SCRATCH 8

/* max number of rings */ -#define RADEON_NUM_RINGS 3 +#define RADEON_NUM_RINGS 3 + +/* fence seq are set to this number when signaled */ +#define RADEON_FENCE_SIGNALED_SEQ 0LL +#define RADEON_FENCE_NOTEMITED_SEQ (~0LL)

/* internal ring indices */ /* r1xx+ has gfx CP ring */ -#define RADEON_RING_TYPE_GFX_INDEX 0 +#define RADEON_RING_TYPE_GFX_INDEX 0

/* cayman has 2 compute CP rings */ -#define CAYMAN_RING_TYPE_CP1_INDEX 1 -#define CAYMAN_RING_TYPE_CP2_INDEX 2 +#define CAYMAN_RING_TYPE_CP1_INDEX 1 +#define CAYMAN_RING_TYPE_CP2_INDEX 2

/* hardcode those limit for now */ -#define RADEON_VA_RESERVED_SIZE (8 << 20) -#define RADEON_IB_VM_MAX_SIZE (64 << 10) +#define RADEON_VA_RESERVED_SIZE (8 << 20) +#define RADEON_IB_VM_MAX_SIZE (64 << 10)

/* * Errata workarounds. @@ -254,8 +258,9 @@ struct radeon_fence_driver { uint32_t scratch_reg; uint64_t gpu_addr; volatile uint32_t *cpu_addr; - atomic_t seq; - uint32_t last_seq; + /* seq is protected by ring emission lock */ + uint64_t seq; + atomic64_t last_seq; unsigned long last_activity; wait_queue_head_t queue; struct list_head emitted; @@ -268,11 +273,9 @@ struct radeon_fence { struct kref kref; struct list_head list; /* protected by radeon_fence.lock */ - uint32_t seq; - bool emitted; - bool signaled; + uint64_t seq; /* RB, DMA, etc. */ - int ring; + unsigned ring; struct radeon_semaphore *semaphore; };

diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 5bb78bf..feb2bbc 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -66,14 +66,14 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) unsigned long irq_flags;

write_lock_irqsave(&rdev->fence_lock, irq_flags); - if (fence->emitted) { + if (fence->seq && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; } - fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq); + /* we are protected by the ring emission mutex */ + fence->seq = ++rdev->fence_drv[fence->ring].seq; radeon_fence_ring_emit(rdev, fence->ring, fence); trace_radeon_fence_emit(rdev->ddev, fence->seq); - fence->emitted = true; /* are we the first fence on a previusly idle ring? */ if (list_empty(&rdev->fence_drv[fence->ring].emitted)) { rdev->fence_drv[fence->ring].last_activity = jiffies; @@ -87,14 +87,60 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring) { struct radeon_fence *fence; struct list_head *i, *n; - uint32_t seq; + uint64_t seq, last_seq; + unsigned count_loop = 0; bool wake = false;

- seq = radeon_fence_read(rdev, ring); - if (seq == rdev->fence_drv[ring].last_seq) - return false; + /* Note there is a scenario here for an infinite loop but it's + * very unlikely to happen. For it to happen, the current polling + * process need to be interrupted by another process and another + * process needs to update the last_seq btw the atomic read and + * xchg of the current process. + * + * More over for this to go in infinite loop there need to be + * continuously new fence signaled ie radeon_fence_read needs + * to return a different value each time for both the currently + * polling process and the other process that xchg the last_seq + * btw atomic read and xchg of the current process. And the + * value the other process set as last seq must be higher than + * the seq value we just read. Which means that current process + * need to be interrupted after radeon_fence_read and before + * atomic xchg. + * + * To be even more safe we count the number of time we loop and + * we bail after 10 loop just accepting the fact that we might + * have temporarly set the last_seq not to the true real last + * seq but to an older one. + */ + last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); + do { + seq = radeon_fence_read(rdev, ring); + seq |= last_seq & 0xffffffff00000000LL; + if (seq < last_seq) { + seq += 0x100000000LL; + }

- rdev->fence_drv[ring].last_seq = seq; + if (!wake && seq == last_seq) { + return false; + } + /* If we loop over we don't want to return without + * checking if a fence is signaled as it means that the + * seq we just read is different from the previous on. + */ + wake = true; + if ((count_loop++) > 10) { + /* We looped over too many time leave with the + * fact that we might have set an older fence + * seq then the current real last seq as signaled + * by the hw. + */ + break; + } + last_seq = seq; + } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); + + /* reset wake to false */ + wake = false; rdev->fence_drv[ring].last_activity = jiffies;

n = NULL; @@ -112,7 +158,7 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring) n = i->prev; list_move_tail(i, &rdev->fence_drv[ring].signaled); fence = list_entry(i, struct radeon_fence, list); - fence->signaled = true; + fence->seq = RADEON_FENCE_SIGNALED_SEQ; i = n; } while (i != &rdev->fence_drv[ring].emitted); wake = true; @@ -128,7 +174,7 @@ static void radeon_fence_destroy(struct kref *kref) fence = container_of(kref, struct radeon_fence, kref); write_lock_irqsave(&fence->rdev->fence_lock, irq_flags); list_del(&fence->list); - fence->emitted = false; + fence->seq = RADEON_FENCE_NOTEMITED_SEQ; write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags); if (fence->semaphore) radeon_semaphore_free(fence->rdev, fence->semaphore); @@ -145,9 +191,7 @@ int radeon_fence_create(struct radeon_device *rdev, } kref_init(&((*fence)->kref)); (*fence)->rdev = rdev; - (*fence)->emitted = false; - (*fence)->signaled = false; - (*fence)->seq = 0; + (*fence)->seq = RADEON_FENCE_NOTEMITED_SEQ; (*fence)->ring = ring; (*fence)->semaphore = NULL; INIT_LIST_HEAD(&(*fence)->list); @@ -163,18 +207,18 @@ bool radeon_fence_signaled(struct radeon_fence *fence) return true;

write_lock_irqsave(&fence->rdev->fence_lock, irq_flags); - signaled = fence->signaled; + signaled = (fence->seq == RADEON_FENCE_SIGNALED_SEQ); /* if we are shuting down report all fence as signaled */ if (fence->rdev->shutdown) { signaled = true; } - if (!fence->emitted) { + if (fence->seq == RADEON_FENCE_NOTEMITED_SEQ) { WARN(1, "Querying an unemitted fence : %p !\n", fence); signaled = true; } if (!signaled) { radeon_fence_poll_locked(fence->rdev, fence->ring); - signaled = fence->signaled; + signaled = (fence->seq == RADEON_FENCE_SIGNALED_SEQ); } write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags); return signaled; @@ -183,8 +227,8 @@ bool radeon_fence_signaled(struct radeon_fence *fence) int radeon_fence_wait(struct radeon_fence *fence, bool intr) { struct radeon_device *rdev; - unsigned long irq_flags, timeout; - u32 seq; + unsigned long irq_flags, timeout, last_activity; + uint64_t seq; int i, r; bool signaled;

@@ -207,7 +251,9 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) timeout = 1; } /* save current sequence value used to check for GPU lockups */ - seq = rdev->fence_drv[fence->ring].last_seq; + seq = atomic64_read(&rdev->fence_drv[fence->ring].last_seq); + /* Save current last activity valuee, used to check for GPU lockups */ + last_activity = rdev->fence_drv[fence->ring].last_activity; read_unlock_irqrestore(&rdev->fence_lock, irq_flags);

trace_radeon_fence_wait_begin(rdev->ddev, seq); @@ -235,24 +281,23 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) }

write_lock_irqsave(&rdev->fence_lock, irq_flags); - /* check if sequence value has changed since last_activity */ - if (seq != rdev->fence_drv[fence->ring].last_seq) { + /* test if somebody else has already decided that this is a lockup */ + if (last_activity != rdev->fence_drv[fence->ring].last_activity) { write_unlock_irqrestore(&rdev->fence_lock, irq_flags); continue; }

- /* change sequence value on all rings, so nobody else things there is a lockup */ - for (i = 0; i < RADEON_NUM_RINGS; ++i) - rdev->fence_drv[i].last_seq -= 0x10000; - - rdev->fence_drv[fence->ring].last_activity = jiffies; write_unlock_irqrestore(&rdev->fence_lock, irq_flags);

if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) { - /* good news we believe it's a lockup */ - printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n", - fence->seq, seq); + dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n", + fence->seq, seq); + + /* change last activity so nobody else think there is a lockup */ + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + rdev->fence_drv[i].last_activity = jiffies; + }

/* mark the ring as not ready any more */ rdev->ring[fence->ring].ready = false; @@ -387,9 +432,9 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) } rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; - radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring); + radeon_fence_write(rdev, rdev->fence_drv[ring].seq, ring); rdev->fence_drv[ring].initialized = true; - DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n", + DRM_INFO("fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n", ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; @@ -400,7 +445,8 @@ static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) rdev->fence_drv[ring].scratch_reg = -1; rdev->fence_drv[ring].cpu_addr = NULL; rdev->fence_drv[ring].gpu_addr = 0; - atomic_set(&rdev->fence_drv[ring].seq, 0); + rdev->fence_drv[ring].seq = 0; + atomic64_set(&rdev->fence_drv[ring].last_seq, 0); INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); init_waitqueue_head(&rdev->fence_drv[ring].queue); @@ -458,12 +504,12 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data) continue;

seq_printf(m, "--- ring %d ---\n", i); - seq_printf(m, "Last signaled fence 0x%08X\n", - radeon_fence_read(rdev, i)); + seq_printf(m, "Last signaled fence 0x%016lx\n", + atomic64_read(&rdev->fence_drv[i].last_seq)); if (!list_empty(&rdev->fence_drv[i].emitted)) { fence = list_entry(rdev->fence_drv[i].emitted.prev, struct radeon_fence, list); - seq_printf(m, "Last emitted fence %p with 0x%08X\n", + seq_printf(m, "Last emitted fence %p with 0x%016llx\n", fence, fence->seq); } } diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index a4d60ae..4ae222b 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -82,7 +82,7 @@ bool radeon_ib_try_free(struct radeon_device *rdev, struct radeon_ib *ib) bool done = false;

/* only free ib which have been emited */ - if (ib->fence && ib->fence->emitted) { + if (ib->fence && ib->fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { if (radeon_fence_signaled(ib->fence)) { radeon_fence_unref(&ib->fence); radeon_sa_bo_free(rdev, &ib->sa_bo); @@ -149,8 +149,9 @@ retry: /* this should be rare event, ie all ib scheduled none signaled yet. */ for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { - if (rdev->ib_pool.ibs[idx].fence && rdev->ib_pool.ibs[idx].fence->emitted) { - r = radeon_fence_wait(rdev->ib_pool.ibs[idx].fence, false); + struct radeon_fence *fence = rdev->ib_pool.ibs[idx].fence; + if (fence && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { + r = radeon_fence_wait(fence, false); if (!r) { goto retry; } @@ -173,7 +174,7 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib) return; } radeon_mutex_lock(&rdev->ib_pool.mutex); - if (tmp->fence && !tmp->fence->emitted) { + if (tmp->fence && tmp->fence->seq == RADEON_FENCE_NOTEMITED_SEQ) { radeon_sa_bo_free(rdev, &tmp->sa_bo); radeon_fence_unref(&tmp->fence); }

Some callers illegal called fence_wait_next/empty while holding the ring emission mutex. So don't relock the mutex in that cases, and move the actual locking into the fence code.

v2: Don't try to unlock the mutex if it isn't locked.

Signed-off-by: Christian König deathsimple@vodafone.de --- drivers/gpu/drm/radeon/radeon.h | 4 +-- drivers/gpu/drm/radeon/radeon_device.c | 5 +++- drivers/gpu/drm/radeon/radeon_fence.c | 43 +++++++++++++++++++++----------- drivers/gpu/drm/radeon/radeon_pm.c | 8 +----- drivers/gpu/drm/radeon/radeon_ring.c | 6 +---- 5 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 7c87117..701094b 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -284,8 +284,8 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence); void radeon_fence_process(struct radeon_device *rdev, int ring); bool radeon_fence_signaled(struct radeon_fence *fence); int radeon_fence_wait(struct radeon_fence *fence, bool interruptible); -int radeon_fence_wait_next(struct radeon_device *rdev, int ring); -int radeon_fence_wait_empty(struct radeon_device *rdev, int ring); +int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring); +int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring); struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence); void radeon_fence_unref(struct radeon_fence **fence); unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring); diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 0e7b72a..b827b2e 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -912,9 +912,12 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state) } /* evict vram memory */ radeon_bo_evict_vram(rdev); + + mutex_lock(&rdev->ring_lock); /* wait for gpu to finish processing current batch */ for (i = 0; i < RADEON_NUM_RINGS; i++) - radeon_fence_wait_empty(rdev, i); + radeon_fence_wait_empty_locked(rdev, i); + mutex_unlock(&rdev->ring_lock);

radeon_save_bios_scratch_regs(rdev);

diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index ed20225..098d1fa 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -194,7 +194,7 @@ bool radeon_fence_signaled(struct radeon_fence *fence) }

static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq, - unsigned ring, bool intr) + unsigned ring, bool intr, bool lock_ring) { unsigned long timeout, last_activity; uint64_t seq; @@ -249,8 +249,16 @@ static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq, if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) { continue; } + + if (lock_ring) { + mutex_lock(&rdev->ring_lock); + } + /* test if somebody else has already decided that this is a lockup */ if (last_activity != rdev->fence_drv[ring].last_activity) { + if (lock_ring) { + mutex_unlock(&rdev->ring_lock); + } continue; }

@@ -264,15 +272,17 @@ static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq, rdev->fence_drv[i].last_activity = jiffies; }

- /* change last activity so nobody else think there is a lockup */ - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - rdev->fence_drv[i].last_activity = jiffies; - } - /* mark the ring as not ready any more */ rdev->ring[ring].ready = false; + if (lock_ring) { + mutex_unlock(&rdev->ring_lock); + } return -EDEADLK; } + + if (lock_ring) { + mutex_unlock(&rdev->ring_lock); + } } } return 0; @@ -287,7 +297,8 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) return -EINVAL; }

- r = radeon_fence_wait_seq(fence->rdev, fence->seq, fence->ring, intr); + r = radeon_fence_wait_seq(fence->rdev, fence->seq, + fence->ring, intr, true); if (r) { return r; } @@ -295,7 +306,7 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) return 0; }

-int radeon_fence_wait_next(struct radeon_device *rdev, int ring) +int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring) { uint64_t seq;

@@ -305,20 +316,22 @@ int radeon_fence_wait_next(struct radeon_device *rdev, int ring) */ seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; if (seq >= rdev->fence_drv[ring].seq) { - /* nothing to wait for, last_seq is already the last emited fence */ - return 0; + /* nothing to wait for, last_seq is + already the last emited fence */ + return -ENOENT; } - return radeon_fence_wait_seq(rdev, seq, ring, false); + return radeon_fence_wait_seq(rdev, seq, ring, false, false); }

-int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) +int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring) { /* We are not protected by ring lock when reading current seq * but it's ok as wait empty is call from place where no more * activity can be scheduled so there won't be concurrent access * to seq value. */ - return radeon_fence_wait_seq(rdev, rdev->fence_drv[ring].seq, ring, false); + return radeon_fence_wait_seq(rdev, rdev->fence_drv[ring].seq, + ring, false, false); }

struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) @@ -410,14 +423,16 @@ void radeon_fence_driver_fini(struct radeon_device *rdev) { int ring;

+ mutex_lock(&rdev->ring_lock); for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { if (!rdev->fence_drv[ring].initialized) continue; - radeon_fence_wait_empty(rdev, ring); + radeon_fence_wait_empty_locked(rdev, ring); wake_up_all(&rdev->fence_drv[ring].queue); radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); rdev->fence_drv[ring].initialized = false; } + mutex_unlock(&rdev->ring_lock); }

diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c index 7c38745..0882554 100644 --- a/drivers/gpu/drm/radeon/radeon_pm.c +++ b/drivers/gpu/drm/radeon/radeon_pm.c @@ -270,13 +270,7 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev) } else { struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; if (ring->ready) { - struct radeon_fence *fence; - radeon_ring_alloc(rdev, ring, 64); - radeon_fence_create(rdev, &fence, radeon_ring_index(rdev, ring)); - radeon_fence_emit(rdev, fence); - radeon_ring_commit(rdev, ring); - radeon_fence_wait(fence, false); - radeon_fence_unref(&fence); + radeon_fence_wait_empty_locked(rdev, RADEON_RING_TYPE_GFX_INDEX); } } radeon_unmap_vram_bos(rdev); diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index 4ae222b..2fdc8c3 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -347,9 +347,7 @@ int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsi if (ndw < ring->ring_free_dw) { break; } - mutex_unlock(&rdev->ring_lock); - r = radeon_fence_wait_next(rdev, radeon_ring_index(rdev, ring)); - mutex_lock(&rdev->ring_lock); + r = radeon_fence_wait_next_locked(rdev, radeon_ring_index(rdev, ring)); if (r) return r; } @@ -408,7 +406,6 @@ void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring * { int r;

- mutex_lock(&rdev->ring_lock); radeon_ring_free_size(rdev, ring); if (ring->rptr == ring->wptr) { r = radeon_ring_alloc(rdev, ring, 1); @@ -417,7 +414,6 @@ void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring * radeon_ring_commit(rdev, ring); } } - mutex_unlock(&rdev->ring_lock); }

void radeon_ring_lockup_update(struct radeon_ring *ring)

Make the suballocator self containing to locking.

v2: split the bugfix into a seperate patch. v3: remove some unreleated changes.

Sig-off-by: Christian König deathsimple@vodafone.de --- drivers/gpu/drm/radeon/radeon.h | 1 + drivers/gpu/drm/radeon/radeon_sa.c | 6 ++++++ 2 files changed, 7 insertions(+)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 701094b..8a6b1b3 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -381,6 +381,7 @@ struct radeon_bo_list { * alignment). */ struct radeon_sa_manager { + spinlock_t lock; struct radeon_bo *bo; struct list_head sa_bo; unsigned size; diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c index 8fbfe69..aed0a8c 100644 --- a/drivers/gpu/drm/radeon/radeon_sa.c +++ b/drivers/gpu/drm/radeon/radeon_sa.c @@ -37,6 +37,7 @@ int radeon_sa_bo_manager_init(struct radeon_device *rdev, { int r;

+ spin_lock_init(&sa_manager->lock); sa_manager->bo = NULL; sa_manager->size = size; sa_manager->domain = domain; @@ -139,6 +140,7 @@ int radeon_sa_bo_new(struct radeon_device *rdev,

BUG_ON(align > RADEON_GPU_PAGE_SIZE); BUG_ON(size > sa_manager->size); + spin_lock(&sa_manager->lock);

/* no one ? */ head = sa_manager->sa_bo.prev; @@ -172,6 +174,7 @@ int radeon_sa_bo_new(struct radeon_device *rdev, offset += wasted; if ((sa_manager->size - offset) < size) { /* failed to find somethings big enough */ + spin_unlock(&sa_manager->lock); return -ENOMEM; }

@@ -180,10 +183,13 @@ out: sa_bo->offset = offset; sa_bo->size = size; list_add(&sa_bo->list, head); + spin_unlock(&sa_manager->lock); return 0; }

void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo *sa_bo) { + spin_lock(&sa_bo->manager->lock); list_del_init(&sa_bo->list); + spin_unlock(&sa_bo->manager->lock); }

Instead of offset + size keep start and end offset directly.

Signed-off-by: Christian König deathsimple@vodafone.de --- drivers/gpu/drm/radeon/radeon.h | 4 ++-- drivers/gpu/drm/radeon/radeon_cs.c | 4 ++-- drivers/gpu/drm/radeon/radeon_object.h | 4 ++-- drivers/gpu/drm/radeon/radeon_sa.c | 13 +++++++------ 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 8a6b1b3..d1c2154 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -396,8 +396,8 @@ struct radeon_sa_bo; struct radeon_sa_bo { struct list_head list; struct radeon_sa_manager *manager; - unsigned offset; - unsigned size; + unsigned soffset; + unsigned eoffset; };

/* diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index 289b0d7..b778037 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -477,7 +477,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, /* ib pool is bind at 0 in virtual address space to gpu_addr is the * offset inside the pool bo */ - parser->const_ib->gpu_addr = parser->const_ib->sa_bo.offset; + parser->const_ib->gpu_addr = parser->const_ib->sa_bo.soffset; r = radeon_ib_schedule(rdev, parser->const_ib); if (r) goto out; @@ -487,7 +487,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, /* ib pool is bind at 0 in virtual address space to gpu_addr is the * offset inside the pool bo */ - parser->ib->gpu_addr = parser->ib->sa_bo.offset; + parser->ib->gpu_addr = parser->ib->sa_bo.soffset; parser->ib->is_const_ib = false; r = radeon_ib_schedule(rdev, parser->ib); out: diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h index d9fca1e..99ab46a 100644 --- a/drivers/gpu/drm/radeon/radeon_object.h +++ b/drivers/gpu/drm/radeon/radeon_object.h @@ -149,12 +149,12 @@ extern struct radeon_bo_va *radeon_bo_va(struct radeon_bo *rbo,

static inline uint64_t radeon_sa_bo_gpu_addr(struct radeon_sa_bo *sa_bo) { - return sa_bo->manager->gpu_addr + sa_bo->offset; + return sa_bo->manager->gpu_addr + sa_bo->soffset; }

static inline void * radeon_sa_bo_cpu_addr(struct radeon_sa_bo *sa_bo) { - return sa_bo->manager->cpu_ptr + sa_bo->offset; + return sa_bo->manager->cpu_ptr + sa_bo->soffset; }

extern int radeon_sa_bo_manager_init(struct radeon_device *rdev, diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c index 1db0568..3bea7ba 100644 --- a/drivers/gpu/drm/radeon/radeon_sa.c +++ b/drivers/gpu/drm/radeon/radeon_sa.c @@ -152,11 +152,11 @@ int radeon_sa_bo_new(struct radeon_device *rdev, offset = 0; list_for_each_entry(tmp, &sa_manager->sa_bo, list) { /* room before this object ? */ - if (offset < tmp->offset && (tmp->offset - offset) >= size) { + if (offset < tmp->soffset && (tmp->soffset - offset) >= size) { head = tmp->list.prev; goto out; } - offset = tmp->offset + tmp->size; + offset = tmp->eoffset; wasted = offset % align; if (wasted) { wasted = align - wasted; @@ -166,7 +166,7 @@ int radeon_sa_bo_new(struct radeon_device *rdev, /* room at the end ? */ head = sa_manager->sa_bo.prev; tmp = list_entry(head, struct radeon_sa_bo, list); - offset = tmp->offset + tmp->size; + offset = tmp->eoffset; wasted = offset % align; if (wasted) { wasted = align - wasted; @@ -180,8 +180,8 @@ int radeon_sa_bo_new(struct radeon_device *rdev,

out: sa_bo->manager = sa_manager; - sa_bo->offset = offset; - sa_bo->size = size; + sa_bo->soffset = offset; + sa_bo->eoffset = offset + size; list_add(&sa_bo->list, head); spin_unlock(&sa_manager->lock); return 0; @@ -202,7 +202,8 @@ void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager,

spin_lock(&sa_manager->lock); list_for_each_entry(i, &sa_manager->sa_bo, list) { - seq_printf(m, "offset %08d: size %4d\n", i->offset, i->size); + seq_printf(m, "[%08x %08x] size %4d [%p]\n", + i->soffset, i->eoffset, i->eoffset - i->soffset, i); } spin_unlock(&sa_manager->lock); }

Define the interface without modifying the allocation algorithm in any way.

v2: rebase on top of fence new uint64 patch v3: add ring to debugfs output

Signed-off-by: Jerome Glisse jglisse@redhat.com Signed-off-by: Christian König deathsimple@vodafone.de --- drivers/gpu/drm/radeon/radeon.h | 1 + drivers/gpu/drm/radeon/radeon_gart.c | 6 +-- drivers/gpu/drm/radeon/radeon_object.h | 5 ++- drivers/gpu/drm/radeon/radeon_ring.c | 8 ++-- drivers/gpu/drm/radeon/radeon_sa.c | 60 ++++++++++++++++++++++++----- drivers/gpu/drm/radeon/radeon_semaphore.c | 2 +- 6 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 9374ab1..ada70d1 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -398,6 +398,7 @@ struct radeon_sa_bo { struct radeon_sa_manager *manager; unsigned soffset; unsigned eoffset; + struct radeon_fence *fence; };

/* diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c index c5789ef..53dba8e 100644 --- a/drivers/gpu/drm/radeon/radeon_gart.c +++ b/drivers/gpu/drm/radeon/radeon_gart.c @@ -326,7 +326,7 @@ static void radeon_vm_unbind_locked(struct radeon_device *rdev, rdev->vm_manager.use_bitmap &= ~(1 << vm->id); list_del_init(&vm->list); vm->id = -1; - radeon_sa_bo_free(rdev, &vm->sa_bo); + radeon_sa_bo_free(rdev, &vm->sa_bo, NULL); vm->pt = NULL;

list_for_each_entry(bo_va, &vm->va, vm_list) { @@ -395,7 +395,7 @@ int radeon_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm) retry: r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, &vm->sa_bo, RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8), - RADEON_GPU_PAGE_SIZE); + RADEON_GPU_PAGE_SIZE, false); if (r) { if (list_empty(&rdev->vm_manager.lru_vm)) { return r; @@ -426,7 +426,7 @@ retry_id: /* do hw bind */ r = rdev->vm_manager.funcs->bind(rdev, vm, id); if (r) { - radeon_sa_bo_free(rdev, &vm->sa_bo); + radeon_sa_bo_free(rdev, &vm->sa_bo, NULL); return r; } rdev->vm_manager.use_bitmap |= 1 << id; diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h index 4fc7f07..befec7d 100644 --- a/drivers/gpu/drm/radeon/radeon_object.h +++ b/drivers/gpu/drm/radeon/radeon_object.h @@ -169,9 +169,10 @@ extern int radeon_sa_bo_manager_suspend(struct radeon_device *rdev, extern int radeon_sa_bo_new(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, struct radeon_sa_bo **sa_bo, - unsigned size, unsigned align); + unsigned size, unsigned align, bool block); extern void radeon_sa_bo_free(struct radeon_device *rdev, - struct radeon_sa_bo **sa_bo); + struct radeon_sa_bo **sa_bo, + struct radeon_fence *fence); #if defined(CONFIG_DEBUG_FS) extern void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager, struct seq_file *m); diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index 45adb37..1748d93 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -85,7 +85,7 @@ bool radeon_ib_try_free(struct radeon_device *rdev, struct radeon_ib *ib) if (ib->fence && ib->fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { if (radeon_fence_signaled(ib->fence)) { radeon_fence_unref(&ib->fence); - radeon_sa_bo_free(rdev, &ib->sa_bo); + radeon_sa_bo_free(rdev, &ib->sa_bo, NULL); done = true; } } @@ -124,7 +124,7 @@ retry: if (rdev->ib_pool.ibs[idx].fence == NULL) { r = radeon_sa_bo_new(rdev, &rdev->ib_pool.sa_manager, &rdev->ib_pool.ibs[idx].sa_bo, - size, 256); + size, 256, false); if (!r) { *ib = &rdev->ib_pool.ibs[idx]; (*ib)->ptr = radeon_sa_bo_cpu_addr((*ib)->sa_bo); @@ -173,7 +173,7 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib) } radeon_mutex_lock(&rdev->ib_pool.mutex); if (tmp->fence && tmp->fence->seq == RADEON_FENCE_NOTEMITED_SEQ) { - radeon_sa_bo_free(rdev, &tmp->sa_bo); + radeon_sa_bo_free(rdev, &tmp->sa_bo, NULL); radeon_fence_unref(&tmp->fence); } radeon_mutex_unlock(&rdev->ib_pool.mutex); @@ -247,7 +247,7 @@ void radeon_ib_pool_fini(struct radeon_device *rdev) radeon_mutex_lock(&rdev->ib_pool.mutex); if (rdev->ib_pool.ready) { for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { - radeon_sa_bo_free(rdev, &rdev->ib_pool.ibs[i].sa_bo); + radeon_sa_bo_free(rdev, &rdev->ib_pool.ibs[i].sa_bo, NULL); radeon_fence_unref(&rdev->ib_pool.ibs[i].fence); } radeon_sa_bo_manager_fini(rdev, &rdev->ib_pool.sa_manager); diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c index 625f2d4..90ee8ad 100644 --- a/drivers/gpu/drm/radeon/radeon_sa.c +++ b/drivers/gpu/drm/radeon/radeon_sa.c @@ -129,20 +129,32 @@ int radeon_sa_bo_manager_suspend(struct radeon_device *rdev, * * Alignment can't be bigger than page size */ + +static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo) +{ + list_del(&sa_bo->list); + radeon_fence_unref(&sa_bo->fence); + kfree(sa_bo); +} + int radeon_sa_bo_new(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, struct radeon_sa_bo **sa_bo, - unsigned size, unsigned align) + unsigned size, unsigned align, bool block) { - struct radeon_sa_bo *tmp; + struct radeon_fence *fence = NULL; + struct radeon_sa_bo *tmp, *next; struct list_head *head; unsigned offset = 0, wasted = 0; + int r;

BUG_ON(align > RADEON_GPU_PAGE_SIZE); BUG_ON(size > sa_manager->size);

*sa_bo = kmalloc(sizeof(struct radeon_sa_bo), GFP_KERNEL);

+retry: + spin_lock(&sa_manager->lock);

/* no one ? */ @@ -153,7 +165,17 @@ int radeon_sa_bo_new(struct radeon_device *rdev,

/* look for a hole big enough */ offset = 0; - list_for_each_entry(tmp, &sa_manager->sa_bo, list) { + list_for_each_entry_safe(tmp, next, &sa_manager->sa_bo, list) { + /* try to free this object */ + if (tmp->fence) { + if (radeon_fence_signaled(tmp->fence)) { + radeon_sa_bo_remove_locked(tmp); + continue; + } else { + fence = tmp->fence; + } + } + /* room before this object ? */ if (offset < tmp->soffset && (tmp->soffset - offset) >= size) { head = tmp->list.prev; @@ -178,6 +200,13 @@ int radeon_sa_bo_new(struct radeon_device *rdev, if ((sa_manager->size - offset) < size) { /* failed to find somethings big enough */ spin_unlock(&sa_manager->lock); + if (block && fence) { + r = radeon_fence_wait(fence, false); + if (r) + return r; + + goto retry; + } kfree(*sa_bo); *sa_bo = NULL; return -ENOMEM; @@ -192,15 +221,22 @@ out: return 0; }

-void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo) +void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo, + struct radeon_fence *fence) { + struct radeon_sa_manager *sa_manager; + if (!sa_bo || !*sa_bo) return;

- spin_lock(&(*sa_bo)->manager->lock); - list_del_init(&(*sa_bo)->list); - spin_unlock(&(*sa_bo)->manager->lock); - kfree(*sa_bo); + sa_manager = (*sa_bo)->manager; + spin_lock(&sa_manager->lock); + if (fence && fence->seq && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { + (*sa_bo)->fence = radeon_fence_ref(fence); + } else { + radeon_sa_bo_remove_locked(*sa_bo); + } + spin_unlock(&sa_manager->lock); *sa_bo = NULL; }

@@ -212,8 +248,14 @@ void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager,

spin_lock(&sa_manager->lock); list_for_each_entry(i, &sa_manager->sa_bo, list) { - seq_printf(m, "[%08x %08x] size %4d [%p]\n", + seq_printf(m, "[%08x %08x] size %4d (%p)", i->soffset, i->eoffset, i->eoffset - i->soffset, i); + if (i->fence) { + seq_printf(m, " protected by %Ld (%p) on ring %d\n", + i->fence->seq, i->fence, i->fence->ring); + } else { + seq_printf(m, "\n"); + } } spin_unlock(&sa_manager->lock); } diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c index d518d32..dbde874 100644 --- a/drivers/gpu/drm/radeon/radeon_semaphore.c +++ b/drivers/gpu/drm/radeon/radeon_semaphore.c @@ -72,7 +72,7 @@ static int radeon_semaphore_add_bo(struct radeon_device *rdev) static void radeon_semaphore_del_bo_locked(struct radeon_device *rdev, struct radeon_semaphore_bo *bo) { - radeon_sa_bo_free(rdev, &bo->ib->sa_bo); + radeon_sa_bo_free(rdev, &bo->ib->sa_bo, NULL); radeon_fence_unref(&bo->ib->fence); list_del(&bo->list); kfree(bo);

A startover with a new idea for a multiple ring allocator. Should perform as well as a normal ring allocator as long as only one ring does somthing, but falls back to a more complex algorithm if more complex things start to happen.

We store the last allocated bo in last, we always try to allocate after the last allocated bo. Principle is that in a linear GPU ring progression was is after last is the oldest bo we allocated and thus the first one that should no longer be in use by the GPU.

If it's not the case we skip over the bo after last to the closest done bo if such one exist. If none exist and we are not asked to block we report failure to allocate.

If we are asked to block we wait on all the oldest fence of all rings. We just wait for any of those fence to complete.

v2: We need to be able to let hole point to the list_head, otherwise try free will never free the first allocation of the list. Also stop calling radeon_fence_signalled more than necessary.

v3: Don't free allocations without considering them as a hole, otherwise we might lose holes. Also return ENOMEM instead of ENOENT when running out of fences to wait for. Limit the number of holes we try for each ring to 3.

Signed-off-by: Christian König deathsimple@vodafone.de Signed-off-by: Jerome Glisse jglisse@redhat.com --- drivers/gpu/drm/radeon/radeon.h | 7 +- drivers/gpu/drm/radeon/radeon_ring.c | 19 +-- drivers/gpu/drm/radeon/radeon_sa.c | 312 ++++++++++++++++++++++++---------- 3 files changed, 231 insertions(+), 107 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 37a7459..cc7f16a 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -385,7 +385,9 @@ struct radeon_bo_list { struct radeon_sa_manager { spinlock_t lock; struct radeon_bo *bo; - struct list_head sa_bo; + struct list_head *hole; + struct list_head flist[RADEON_NUM_RINGS]; + struct list_head olist; unsigned size; uint64_t gpu_addr; void *cpu_ptr; @@ -396,7 +398,8 @@ struct radeon_sa_bo;

/* sub-allocation buffer */ struct radeon_sa_bo { - struct list_head list; + struct list_head olist; + struct list_head flist; struct radeon_sa_manager *manager; unsigned soffset; unsigned eoffset; diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index 1748d93..e074ff5 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -204,25 +204,22 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)

int radeon_ib_pool_init(struct radeon_device *rdev) { - struct radeon_sa_manager tmp; int i, r;

- r = radeon_sa_bo_manager_init(rdev, &tmp, - RADEON_IB_POOL_SIZE*64*1024, - RADEON_GEM_DOMAIN_GTT); - if (r) { - return r; - } - radeon_mutex_lock(&rdev->ib_pool.mutex); if (rdev->ib_pool.ready) { radeon_mutex_unlock(&rdev->ib_pool.mutex); - radeon_sa_bo_manager_fini(rdev, &tmp); return 0; }

- rdev->ib_pool.sa_manager = tmp; - INIT_LIST_HEAD(&rdev->ib_pool.sa_manager.sa_bo); + r = radeon_sa_bo_manager_init(rdev, &rdev->ib_pool.sa_manager, + RADEON_IB_POOL_SIZE*64*1024, + RADEON_GEM_DOMAIN_GTT); + if (r) { + radeon_mutex_unlock(&rdev->ib_pool.mutex); + return r; + } + for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { rdev->ib_pool.ibs[i].fence = NULL; rdev->ib_pool.ibs[i].idx = i; diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c index 90ee8ad..c3ac7f4 100644 --- a/drivers/gpu/drm/radeon/radeon_sa.c +++ b/drivers/gpu/drm/radeon/radeon_sa.c @@ -27,21 +27,42 @@ * Authors: * Jerome Glisse glisse@freedesktop.org */ +/* Algorithm: + * + * We store the last allocated bo in "hole", we always try to allocate + * after the last allocated bo. Principle is that in a linear GPU ring + * progression was is after last is the oldest bo we allocated and thus + * the first one that should no longer be in use by the GPU. + * + * If it's not the case we skip over the bo after last to the closest + * done bo if such one exist. If none exist and we are not asked to + * block we report failure to allocate. + * + * If we are asked to block we wait on all the oldest fence of all + * rings. We just wait for any of those fence to complete. + */ #include "drmP.h" #include "drm.h" #include "radeon.h"

+static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo); +static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager); + int radeon_sa_bo_manager_init(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, unsigned size, u32 domain) { - int r; + int i, r;

spin_lock_init(&sa_manager->lock); sa_manager->bo = NULL; sa_manager->size = size; sa_manager->domain = domain; - INIT_LIST_HEAD(&sa_manager->sa_bo); + sa_manager->hole = &sa_manager->olist; + INIT_LIST_HEAD(&sa_manager->olist); + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + INIT_LIST_HEAD(&sa_manager->flist[i]); + }

r = radeon_bo_create(rdev, size, RADEON_GPU_PAGE_SIZE, true, RADEON_GEM_DOMAIN_CPU, &sa_manager->bo); @@ -58,11 +79,15 @@ void radeon_sa_bo_manager_fini(struct radeon_device *rdev, { struct radeon_sa_bo *sa_bo, *tmp;

- if (!list_empty(&sa_manager->sa_bo)) { - dev_err(rdev->dev, "sa_manager is not empty, clearing anyway\n"); + if (!list_empty(&sa_manager->olist)) { + sa_manager->hole = &sa_manager->olist, + radeon_sa_bo_try_free(sa_manager); + if (!list_empty(&sa_manager->olist)) { + dev_err(rdev->dev, "sa_manager is not empty, clearing anyway\n"); + } } - list_for_each_entry_safe(sa_bo, tmp, &sa_manager->sa_bo, list) { - list_del_init(&sa_bo->list); + list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) { + radeon_sa_bo_remove_locked(sa_bo); } radeon_bo_unref(&sa_manager->bo); sa_manager->size = 0; @@ -114,111 +139,203 @@ int radeon_sa_bo_manager_suspend(struct radeon_device *rdev, return r; }

-/* - * Principe is simple, we keep a list of sub allocation in offset - * order (first entry has offset == 0, last entry has the highest - * offset). - * - * When allocating new object we first check if there is room at - * the end total_size - (last_object_offset + last_object_size) >= - * alloc_size. If so we allocate new object there. - * - * When there is not enough room at the end, we start waiting for - * each sub object until we reach object_offset+object_size >= - * alloc_size, this object then become the sub object we return. - * - * Alignment can't be bigger than page size - */ - static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo) { - list_del(&sa_bo->list); + struct radeon_sa_manager *sa_manager = sa_bo->manager; + if (sa_manager->hole == &sa_bo->olist) { + sa_manager->hole = sa_bo->olist.prev; + } + list_del_init(&sa_bo->olist); + list_del_init(&sa_bo->flist); radeon_fence_unref(&sa_bo->fence); kfree(sa_bo); }

+static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager) +{ + struct radeon_sa_bo *sa_bo, *tmp; + + if (sa_manager->hole->next == &sa_manager->olist) + return; + + sa_bo = list_entry(sa_manager->hole->next, struct radeon_sa_bo, olist); + list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) { + if (sa_bo->fence == NULL || !radeon_fence_signaled(sa_bo->fence)) { + return; + } + radeon_sa_bo_remove_locked(sa_bo); + } +} + +static inline unsigned radeon_sa_bo_hole_soffset(struct radeon_sa_manager *sa_manager) +{ + struct list_head *hole = sa_manager->hole; + + if (hole != &sa_manager->olist) { + return list_entry(hole, struct radeon_sa_bo, olist)->eoffset; + } + return 0; +} + +static inline unsigned radeon_sa_bo_hole_eoffset(struct radeon_sa_manager *sa_manager) +{ + struct list_head *hole = sa_manager->hole; + + if (hole->next != &sa_manager->olist) { + return list_entry(hole->next, struct radeon_sa_bo, olist)->soffset; + } + return sa_manager->size; +} + +static bool radeon_sa_bo_try_alloc(struct radeon_sa_manager *sa_manager, + struct radeon_sa_bo *sa_bo, + unsigned size, unsigned align) +{ + unsigned soffset, eoffset, wasted; + + soffset = radeon_sa_bo_hole_soffset(sa_manager); + eoffset = radeon_sa_bo_hole_eoffset(sa_manager); + wasted = (align - (soffset % align)) % align; + + if ((eoffset - soffset) >= (size + wasted)) { + soffset += wasted; + + sa_bo->manager = sa_manager; + sa_bo->soffset = soffset; + sa_bo->eoffset = soffset + size; + list_add(&sa_bo->olist, sa_manager->hole); + INIT_LIST_HEAD(&sa_bo->flist); + sa_manager->hole = &sa_bo->olist; + return true; + } + return false; +} + +static bool radeon_sa_bo_next_hole(struct radeon_sa_manager *sa_manager, + struct radeon_fence **fences, + unsigned *tries) +{ + struct radeon_sa_bo *best_bo = NULL; + unsigned i, soffset, best, tmp; + + /* if hole points to the end of the buffer */ + if (sa_manager->hole->next == &sa_manager->olist) { + /* try again with its beginning */ + sa_manager->hole = &sa_manager->olist; + return true; + } + + soffset = radeon_sa_bo_hole_soffset(sa_manager); + /* to handle wrap around we add sa_manager->size */ + best = sa_manager->size * 2; + /* go over all fence list and try to find the closest sa_bo + * of the current last + */ + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + struct radeon_sa_bo *sa_bo; + + if (list_empty(&sa_manager->flist[i])) { + continue; + } + + sa_bo = list_first_entry(&sa_manager->flist[i], + struct radeon_sa_bo, flist); + + if (!radeon_fence_signaled(sa_bo->fence)) { + fences[i] = sa_bo->fence; + continue; + } + + /* limit the number of tries each ring gets */ + if (tries[i] > 2) { + continue; + } + + tmp = sa_bo->soffset; + if (tmp < soffset) { + /* wrap around, pretend it's after */ + tmp += sa_manager->size; + } + tmp -= soffset; + if (tmp < best) { + /* this sa bo is the closest one */ + best = tmp; + best_bo = sa_bo; + } + } + + if (best_bo) { + ++tries[best_bo->fence->ring]; + sa_manager->hole = best_bo->olist.prev; + + /* we knew that this one is signaled, + so it's save to remote it */ + radeon_sa_bo_remove_locked(best_bo); + return true; + } + return false; +} + int radeon_sa_bo_new(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager, struct radeon_sa_bo **sa_bo, unsigned size, unsigned align, bool block) { - struct radeon_fence *fence = NULL; - struct radeon_sa_bo *tmp, *next; - struct list_head *head; - unsigned offset = 0, wasted = 0; - int r; + struct radeon_fence *fences[RADEON_NUM_RINGS]; + unsigned tries[RADEON_NUM_RINGS]; + int i, r = -ENOMEM;

BUG_ON(align > RADEON_GPU_PAGE_SIZE); BUG_ON(size > sa_manager->size);

*sa_bo = kmalloc(sizeof(struct radeon_sa_bo), GFP_KERNEL); - -retry: + if ((*sa_bo) == NULL) { + return -ENOMEM; + } + (*sa_bo)->manager = sa_manager; + (*sa_bo)->fence = NULL; + INIT_LIST_HEAD(&(*sa_bo)->olist); + INIT_LIST_HEAD(&(*sa_bo)->flist);

spin_lock(&sa_manager->lock); + do { + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + fences[i] = NULL; + tries[i] = 0; + }

- /* no one ? */ - head = sa_manager->sa_bo.prev; - if (list_empty(&sa_manager->sa_bo)) { - goto out; - } + do { + radeon_sa_bo_try_free(sa_manager);

- /* look for a hole big enough */ - offset = 0; - list_for_each_entry_safe(tmp, next, &sa_manager->sa_bo, list) { - /* try to free this object */ - if (tmp->fence) { - if (radeon_fence_signaled(tmp->fence)) { - radeon_sa_bo_remove_locked(tmp); - continue; - } else { - fence = tmp->fence; + if (radeon_sa_bo_try_alloc(sa_manager, *sa_bo, + size, align)) { + spin_unlock(&sa_manager->lock); + return 0; } - }

- /* room before this object ? */ - if (offset < tmp->soffset && (tmp->soffset - offset) >= size) { - head = tmp->list.prev; - goto out; - } - offset = tmp->eoffset; - wasted = offset % align; - if (wasted) { - wasted = align - wasted; - } - offset += wasted; - } - /* room at the end ? */ - head = sa_manager->sa_bo.prev; - tmp = list_entry(head, struct radeon_sa_bo, list); - offset = tmp->eoffset; - wasted = offset % align; - if (wasted) { - wasted = align - wasted; - } - offset += wasted; - if ((sa_manager->size - offset) < size) { - /* failed to find somethings big enough */ - spin_unlock(&sa_manager->lock); - if (block && fence) { - r = radeon_fence_wait(fence, false); - if (r) - return r; - - goto retry; + /* see if we can skip over some allocations */ + } while (radeon_sa_bo_next_hole(sa_manager, fences, tries)); + + if (block) { + spin_unlock(&sa_manager->lock); + r = radeon_fence_wait_any(rdev, fences, false); + spin_lock(&sa_manager->lock); + if (r) { + /* if we have nothing to wait for we + are practically out of memory */ + if (r == -ENOENT) { + r = -ENOMEM; + } + goto out_err; + } } - kfree(*sa_bo); - *sa_bo = NULL; - return -ENOMEM; - } + } while (block);

-out: - (*sa_bo)->manager = sa_manager; - (*sa_bo)->soffset = offset; - (*sa_bo)->eoffset = offset + size; - list_add(&(*sa_bo)->list, head); +out_err: spin_unlock(&sa_manager->lock); - return 0; + kfree(*sa_bo); + *sa_bo = NULL; + return r; }

void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo, @@ -226,13 +343,16 @@ void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo, { struct radeon_sa_manager *sa_manager;

- if (!sa_bo || !*sa_bo) + if (sa_bo == NULL || *sa_bo == NULL) { return; + }

sa_manager = (*sa_bo)->manager; spin_lock(&sa_manager->lock); if (fence && fence->seq && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { (*sa_bo)->fence = radeon_fence_ref(fence); + list_add_tail(&(*sa_bo)->flist, + &sa_manager->flist[fence->ring]); } else { radeon_sa_bo_remove_locked(*sa_bo); } @@ -247,15 +367,19 @@ void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager, struct radeon_sa_bo *i;

spin_lock(&sa_manager->lock); - list_for_each_entry(i, &sa_manager->sa_bo, list) { - seq_printf(m, "[%08x %08x] size %4d (%p)", - i->soffset, i->eoffset, i->eoffset - i->soffset, i); - if (i->fence) { - seq_printf(m, " protected by %Ld (%p) on ring %d\n", - i->fence->seq, i->fence, i->fence->ring); + list_for_each_entry(i, &sa_manager->olist, olist) { + if (&i->olist == sa_manager->hole) { + seq_printf(m, ">"); } else { - seq_printf(m, "\n"); + seq_printf(m, " "); + } + seq_printf(m, "[0x%08x 0x%08x] size %8d", + i->soffset, i->eoffset, i->eoffset - i->soffset); + if (i->fence) { + seq_printf(m, " protected by 0x%016llx on ring %d", + i->fence->seq, i->fence->ring); } + seq_printf(m, "\n"); } spin_unlock(&sa_manager->lock); }

From: Jerome Glisse jglisse@redhat.com

It isn't necessary any more and the suballocator seems to perform even better.

Signed-off-by: Christian König deathsimple@vodafone.de Signed-off-by: Jerome Glisse jglisse@redhat.com --- drivers/gpu/drm/radeon/radeon.h | 17 +- drivers/gpu/drm/radeon/radeon_device.c | 1 - drivers/gpu/drm/radeon/radeon_gart.c | 12 +- drivers/gpu/drm/radeon/radeon_ring.c | 241 ++++++++--------------------- drivers/gpu/drm/radeon/radeon_semaphore.c | 2 +- 5 files changed, 71 insertions(+), 202 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 45164e1..6170307 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -625,7 +625,6 @@ void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);

struct radeon_ib { struct radeon_sa_bo *sa_bo; - unsigned idx; uint32_t length_dw; uint64_t gpu_addr; uint32_t *ptr; @@ -634,18 +633,6 @@ struct radeon_ib { bool is_const_ib; };

-/* - * locking - - * mutex protects scheduled_ibs, ready, alloc_bm - */ -struct radeon_ib_pool { - struct radeon_mutex mutex; - struct radeon_sa_manager sa_manager; - struct radeon_ib ibs[RADEON_IB_POOL_SIZE]; - bool ready; - unsigned head_id; -}; - struct radeon_ring { struct radeon_bo *ring_obj; volatile uint32_t *ring; @@ -787,7 +774,6 @@ struct si_rlc { int radeon_ib_get(struct radeon_device *rdev, int ring, struct radeon_ib **ib, unsigned size); void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib); -bool radeon_ib_try_free(struct radeon_device *rdev, struct radeon_ib *ib); int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib); int radeon_ib_pool_init(struct radeon_device *rdev); void radeon_ib_pool_fini(struct radeon_device *rdev); @@ -1522,7 +1508,8 @@ struct radeon_device { wait_queue_head_t fence_queue; struct mutex ring_lock; struct radeon_ring ring[RADEON_NUM_RINGS]; - struct radeon_ib_pool ib_pool; + bool ib_pool_ready; + struct radeon_sa_manager ring_tmp_bo; struct radeon_irq irq; struct radeon_asic *asic; struct radeon_gem gem; diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 48876c1..e1bc7e9 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -724,7 +724,6 @@ int radeon_device_init(struct radeon_device *rdev, /* mutex initialization are all done here so we * can recall function without having locking issues */ radeon_mutex_init(&rdev->cs_mutex); - radeon_mutex_init(&rdev->ib_pool.mutex); mutex_init(&rdev->ring_lock); mutex_init(&rdev->dc_hw_i2c_mutex); if (rdev->family >= CHIP_R600) diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c index 53dba8e..8e9ef34 100644 --- a/drivers/gpu/drm/radeon/radeon_gart.c +++ b/drivers/gpu/drm/radeon/radeon_gart.c @@ -432,8 +432,8 @@ retry_id: rdev->vm_manager.use_bitmap |= 1 << id; vm->id = id; list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); - return radeon_vm_bo_update_pte(rdev, vm, rdev->ib_pool.sa_manager.bo, - &rdev->ib_pool.sa_manager.bo->tbo.mem); + return radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo, + &rdev->ring_tmp_bo.bo->tbo.mem); }

/* object have to be reserved */ @@ -631,7 +631,7 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) /* map the ib pool buffer at 0 in virtual address space, set * read only */ - r = radeon_vm_bo_add(rdev, vm, rdev->ib_pool.sa_manager.bo, 0, + r = radeon_vm_bo_add(rdev, vm, rdev->ring_tmp_bo.bo, 0, RADEON_VM_PAGE_READABLE | RADEON_VM_PAGE_SNOOPED); return r; } @@ -648,12 +648,12 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) radeon_mutex_unlock(&rdev->cs_mutex);

/* remove all bo */ - r = radeon_bo_reserve(rdev->ib_pool.sa_manager.bo, false); + r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false); if (!r) { - bo_va = radeon_bo_va(rdev->ib_pool.sa_manager.bo, vm); + bo_va = radeon_bo_va(rdev->ring_tmp_bo.bo, vm); list_del_init(&bo_va->bo_list); list_del_init(&bo_va->vm_list); - radeon_bo_unreserve(rdev->ib_pool.sa_manager.bo); + radeon_bo_unreserve(rdev->ring_tmp_bo.bo); kfree(bo_va); } if (!list_empty(&vm->va)) { diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index e074ff5..b3d6942 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -24,6 +24,7 @@ * Authors: Dave Airlie * Alex Deucher * Jerome Glisse + * Christian König */ #include <linux/seq_file.h> #include <linux/slab.h> @@ -33,8 +34,10 @@ #include "radeon.h" #include "atom.h"

-int radeon_debugfs_ib_init(struct radeon_device *rdev); -int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring); +/* + * IB. + */ +int radeon_debugfs_sa_init(struct radeon_device *rdev);

u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx) { @@ -61,106 +64,37 @@ u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx) return idx_value; }

-void radeon_ring_write(struct radeon_ring *ring, uint32_t v) -{ -#if DRM_DEBUG_CODE - if (ring->count_dw <= 0) { - DRM_ERROR("radeon: writting more dword to ring than expected !\n"); - } -#endif - ring->ring[ring->wptr++] = v; - ring->wptr &= ring->ptr_mask; - ring->count_dw--; - ring->ring_free_dw--; -} - -/* - * IB. - */ -bool radeon_ib_try_free(struct radeon_device *rdev, struct radeon_ib *ib) -{ - bool done = false; - - /* only free ib which have been emited */ - if (ib->fence && ib->fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { - if (radeon_fence_signaled(ib->fence)) { - radeon_fence_unref(&ib->fence); - radeon_sa_bo_free(rdev, &ib->sa_bo, NULL); - done = true; - } - } - return done; -} - int radeon_ib_get(struct radeon_device *rdev, int ring, struct radeon_ib **ib, unsigned size) { - struct radeon_fence *fence; - unsigned cretry = 0; - int r = 0, i, idx; - - *ib = NULL; - /* align size on 256 bytes */ - size = ALIGN(size, 256); - - r = radeon_fence_create(rdev, &fence, ring); - if (r) { - dev_err(rdev->dev, "failed to create fence for new IB\n"); - return r; - } + int r;

- radeon_mutex_lock(&rdev->ib_pool.mutex); - idx = rdev->ib_pool.head_id; -retry: - if (cretry > 5) { - dev_err(rdev->dev, "failed to get an ib after 5 retry\n"); - radeon_mutex_unlock(&rdev->ib_pool.mutex); - radeon_fence_unref(&fence); + *ib = kmalloc(sizeof(struct radeon_ib), GFP_KERNEL); + if (*ib == NULL) { return -ENOMEM; } - cretry++; - for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { - radeon_ib_try_free(rdev, &rdev->ib_pool.ibs[idx]); - if (rdev->ib_pool.ibs[idx].fence == NULL) { - r = radeon_sa_bo_new(rdev, &rdev->ib_pool.sa_manager, - &rdev->ib_pool.ibs[idx].sa_bo, - size, 256, false); - if (!r) { - *ib = &rdev->ib_pool.ibs[idx]; - (*ib)->ptr = radeon_sa_bo_cpu_addr((*ib)->sa_bo); - (*ib)->gpu_addr = radeon_sa_bo_gpu_addr((*ib)->sa_bo); - (*ib)->fence = fence; - (*ib)->vm_id = 0; - (*ib)->is_const_ib = false; - /* ib are most likely to be allocated in a ring fashion - * thus rdev->ib_pool.head_id should be the id of the - * oldest ib - */ - rdev->ib_pool.head_id = (1 + idx); - rdev->ib_pool.head_id &= (RADEON_IB_POOL_SIZE - 1); - radeon_mutex_unlock(&rdev->ib_pool.mutex); - return 0; - } - } - idx = (idx + 1) & (RADEON_IB_POOL_SIZE - 1); + r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &(*ib)->sa_bo, size, 256, true); + if (r) { + dev_err(rdev->dev, "failed to get a new IB (%d)\n", r); + kfree(*ib); + *ib = NULL; + return r; } - /* this should be rare event, ie all ib scheduled none signaled yet. - */ - for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { - struct radeon_fence *fence = rdev->ib_pool.ibs[idx].fence; - if (fence && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { - r = radeon_fence_wait(fence, false); - if (!r) { - goto retry; - } - /* an error happened */ - break; - } - idx = (idx + 1) & (RADEON_IB_POOL_SIZE - 1); + r = radeon_fence_create(rdev, &(*ib)->fence, ring); + if (r) { + dev_err(rdev->dev, "failed to create fence for new IB (%d)\n", r); + radeon_sa_bo_free(rdev, &(*ib)->sa_bo, NULL); + kfree(*ib); + *ib = NULL; + return r; } - radeon_mutex_unlock(&rdev->ib_pool.mutex); - radeon_fence_unref(&fence); - return r; + + (*ib)->ptr = radeon_sa_bo_cpu_addr((*ib)->sa_bo); + (*ib)->gpu_addr = radeon_sa_bo_gpu_addr((*ib)->sa_bo); + (*ib)->vm_id = 0; + (*ib)->is_const_ib = false; + + return 0; }

void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib) @@ -171,12 +105,9 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib) if (tmp == NULL) { return; } - radeon_mutex_lock(&rdev->ib_pool.mutex); - if (tmp->fence && tmp->fence->seq == RADEON_FENCE_NOTEMITED_SEQ) { - radeon_sa_bo_free(rdev, &tmp->sa_bo, NULL); - radeon_fence_unref(&tmp->fence); - } - radeon_mutex_unlock(&rdev->ib_pool.mutex); + radeon_sa_bo_free(rdev, &tmp->sa_bo, tmp->fence); + radeon_fence_unref(&tmp->fence); + kfree(tmp); }

int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib) @@ -186,14 +117,14 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)

if (!ib->length_dw || !ring->ready) { /* TODO: Nothings in the ib we should report. */ - DRM_ERROR("radeon: couldn't schedule IB(%u).\n", ib->idx); + dev_err(rdev->dev, "couldn't schedule ib\n"); return -EINVAL; }

/* 64 dwords should be enough for fence too */ r = radeon_ring_lock(rdev, ring, 64); if (r) { - DRM_ERROR("radeon: scheduling IB failed (%d).\n", r); + dev_err(rdev->dev, "scheduling IB failed (%d).\n", r); return r; } radeon_ring_ib_execute(rdev, ib->fence->ring, ib); @@ -204,63 +135,40 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)

int radeon_ib_pool_init(struct radeon_device *rdev) { - int i, r; + int r;

- radeon_mutex_lock(&rdev->ib_pool.mutex); - if (rdev->ib_pool.ready) { - radeon_mutex_unlock(&rdev->ib_pool.mutex); + if (rdev->ib_pool_ready) { return 0; } - - r = radeon_sa_bo_manager_init(rdev, &rdev->ib_pool.sa_manager, + r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo, RADEON_IB_POOL_SIZE*64*1024, RADEON_GEM_DOMAIN_GTT); if (r) { - radeon_mutex_unlock(&rdev->ib_pool.mutex); return r; } - - for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { - rdev->ib_pool.ibs[i].fence = NULL; - rdev->ib_pool.ibs[i].idx = i; - rdev->ib_pool.ibs[i].length_dw = 0; - rdev->ib_pool.ibs[i].sa_bo = NULL; - } - rdev->ib_pool.head_id = 0; - rdev->ib_pool.ready = true; - DRM_INFO("radeon: ib pool ready.\n"); - - if (radeon_debugfs_ib_init(rdev)) { - DRM_ERROR("Failed to register debugfs file for IB !\n"); + rdev->ib_pool_ready = true; + if (radeon_debugfs_sa_init(rdev)) { + dev_err(rdev->dev, "failed to register debugfs file for SA\n"); } - radeon_mutex_unlock(&rdev->ib_pool.mutex); return 0; }

void radeon_ib_pool_fini(struct radeon_device *rdev) { - unsigned i; - - radeon_mutex_lock(&rdev->ib_pool.mutex); - if (rdev->ib_pool.ready) { - for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { - radeon_sa_bo_free(rdev, &rdev->ib_pool.ibs[i].sa_bo, NULL); - radeon_fence_unref(&rdev->ib_pool.ibs[i].fence); - } - radeon_sa_bo_manager_fini(rdev, &rdev->ib_pool.sa_manager); - rdev->ib_pool.ready = false; + if (rdev->ib_pool_ready) { + radeon_sa_bo_manager_fini(rdev, &rdev->ring_tmp_bo); + rdev->ib_pool_ready = false; } - radeon_mutex_unlock(&rdev->ib_pool.mutex); }

int radeon_ib_pool_start(struct radeon_device *rdev) { - return radeon_sa_bo_manager_start(rdev, &rdev->ib_pool.sa_manager); + return radeon_sa_bo_manager_start(rdev, &rdev->ring_tmp_bo); }

int radeon_ib_pool_suspend(struct radeon_device *rdev) { - return radeon_sa_bo_manager_suspend(rdev, &rdev->ib_pool.sa_manager); + return radeon_sa_bo_manager_suspend(rdev, &rdev->ring_tmp_bo); }

int radeon_ib_ring_tests(struct radeon_device *rdev) @@ -296,6 +204,21 @@ int radeon_ib_ring_tests(struct radeon_device *rdev) /* * Ring. */ +int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring); + +void radeon_ring_write(struct radeon_ring *ring, uint32_t v) +{ +#if DRM_DEBUG_CODE + if (ring->count_dw <= 0) { + DRM_ERROR("radeon: writting more dword to ring than expected !\n"); + } +#endif + ring->ring[ring->wptr++] = v; + ring->wptr &= ring->ptr_mask; + ring->count_dw--; + ring->ring_free_dw--; +} + int radeon_ring_index(struct radeon_device *rdev, struct radeon_ring *ring) { /* r1xx-r5xx only has CP ring */ @@ -575,37 +498,13 @@ static struct drm_info_list radeon_debugfs_ring_info_list[] = { {"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp2_index}, };

-static int radeon_debugfs_ib_info(struct seq_file *m, void *data) -{ - struct drm_info_node *node = (struct drm_info_node *) m->private; - struct drm_device *dev = node->minor->dev; - struct radeon_device *rdev = dev->dev_private; - struct radeon_ib *ib = &rdev->ib_pool.ibs[*((unsigned*)node->info_ent->data)]; - unsigned i; - - if (ib == NULL) { - return 0; - } - seq_printf(m, "IB %04u\n", ib->idx); - seq_printf(m, "IB fence %p\n", ib->fence); - seq_printf(m, "IB size %05u dwords\n", ib->length_dw); - for (i = 0; i < ib->length_dw; i++) { - seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]); - } - return 0; -} - -static struct drm_info_list radeon_debugfs_ib_list[RADEON_IB_POOL_SIZE]; -static char radeon_debugfs_ib_names[RADEON_IB_POOL_SIZE][32]; -static unsigned radeon_debugfs_ib_idx[RADEON_IB_POOL_SIZE]; - static int radeon_debugfs_sa_info(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; struct radeon_device *rdev = dev->dev_private;

- radeon_sa_bo_dump_debug_info(&rdev->ib_pool.sa_manager, m); + radeon_sa_bo_dump_debug_info(&rdev->ring_tmp_bo, m);

return 0;

@@ -637,26 +536,10 @@ int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *rin return 0; }

-int radeon_debugfs_ib_init(struct radeon_device *rdev) +int radeon_debugfs_sa_init(struct radeon_device *rdev) { #if defined(CONFIG_DEBUG_FS) - unsigned i; - int r; - - r = radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1); - if (r) - return r; - - for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { - sprintf(radeon_debugfs_ib_names[i], "radeon_ib_%04u", i); - radeon_debugfs_ib_idx[i] = i; - radeon_debugfs_ib_list[i].name = radeon_debugfs_ib_names[i]; - radeon_debugfs_ib_list[i].show = &radeon_debugfs_ib_info; - radeon_debugfs_ib_list[i].driver_features = 0; - radeon_debugfs_ib_list[i].data = &radeon_debugfs_ib_idx[i]; - } - return radeon_debugfs_add_files(rdev, radeon_debugfs_ib_list, - RADEON_IB_POOL_SIZE); + return radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1); #else return 0; #endif diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c index 1bc5513..e2ace5d 100644 --- a/drivers/gpu/drm/radeon/radeon_semaphore.c +++ b/drivers/gpu/drm/radeon/radeon_semaphore.c @@ -41,7 +41,7 @@ int radeon_semaphore_create(struct radeon_device *rdev, if (*semaphore == NULL) { return -ENOMEM; } - r = radeon_sa_bo_new(rdev, &rdev->ib_pool.sa_manager, + r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &(*semaphore)->sa_bo, 8, 8, true); if (r) { kfree(*semaphore);

From: Jerome Glisse jglisse@redhat.com

It never really belonged there in the first place.

Signed-off-by: Christian König deathsimple@vodafone.de --- drivers/gpu/drm/radeon/radeon.h | 16 ++++++++-------- drivers/gpu/drm/radeon/radeon_cs.c | 4 ++-- drivers/gpu/drm/radeon/radeon_fence.c | 3 --- drivers/gpu/drm/radeon/radeon_ring.c | 2 ++ 4 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 6170307..9507be0 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -272,7 +272,6 @@ struct radeon_fence { uint64_t seq; /* RB, DMA, etc. */ unsigned ring; - struct radeon_semaphore *semaphore; };

int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring); @@ -624,13 +623,14 @@ void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc); */

struct radeon_ib { - struct radeon_sa_bo *sa_bo; - uint32_t length_dw; - uint64_t gpu_addr; - uint32_t *ptr; - struct radeon_fence *fence; - unsigned vm_id; - bool is_const_ib; + struct radeon_sa_bo *sa_bo; + uint32_t length_dw; + uint64_t gpu_addr; + uint32_t *ptr; + struct radeon_fence *fence; + unsigned vm_id; + bool is_const_ib; + struct radeon_semaphore *semaphore; };

struct radeon_ring { diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index 5c065bf..dcfe2a0 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -138,12 +138,12 @@ static int radeon_cs_sync_rings(struct radeon_cs_parser *p) return 0; }

- r = radeon_semaphore_create(p->rdev, &p->ib->fence->semaphore); + r = radeon_semaphore_create(p->rdev, &p->ib->semaphore); if (r) { return r; }

- return radeon_semaphore_sync_rings(p->rdev, p->ib->fence->semaphore, + return radeon_semaphore_sync_rings(p->rdev, p->ib->semaphore, sync_to_ring, p->ring); }

diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 3a49311..48ec5e3 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -139,8 +139,6 @@ static void radeon_fence_destroy(struct kref *kref)

fence = container_of(kref, struct radeon_fence, kref); fence->seq = RADEON_FENCE_NOTEMITED_SEQ; - if (fence->semaphore) - radeon_semaphore_free(fence->rdev, fence->semaphore, NULL); kfree(fence); }

@@ -156,7 +154,6 @@ int radeon_fence_create(struct radeon_device *rdev, (*fence)->rdev = rdev; (*fence)->seq = RADEON_FENCE_NOTEMITED_SEQ; (*fence)->ring = ring; - (*fence)->semaphore = NULL; return 0; }

diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index b3d6942..af8e1ee 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -93,6 +93,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring, (*ib)->gpu_addr = radeon_sa_bo_gpu_addr((*ib)->sa_bo); (*ib)->vm_id = 0; (*ib)->is_const_ib = false; + (*ib)->semaphore = NULL;

return 0; } @@ -105,6 +106,7 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib) if (tmp == NULL) { return; } + radeon_semaphore_free(rdev, tmp->semaphore, tmp->fence); radeon_sa_bo_free(rdev, &tmp->sa_bo, tmp->fence); radeon_fence_unref(&tmp->fence); kfree(tmp);

From: Jerome Glisse jglisse@redhat.com

No need to malloc it any more.

Signed-off-by: Jerome Glisse jglisse@redhat.com Signed-off-by: Christian König deathsimple@vodafone.de --- drivers/gpu/drm/radeon/evergreen_cs.c | 10 +++--- drivers/gpu/drm/radeon/r100.c | 38 ++++++++++---------- drivers/gpu/drm/radeon/r200.c | 2 +- drivers/gpu/drm/radeon/r300.c | 4 +-- drivers/gpu/drm/radeon/r600.c | 16 ++++----- drivers/gpu/drm/radeon/r600_cs.c | 22 ++++++------ drivers/gpu/drm/radeon/radeon.h | 8 ++--- drivers/gpu/drm/radeon/radeon_cs.c | 63 ++++++++++++++++----------------- drivers/gpu/drm/radeon/radeon_ring.c | 41 +++++++-------------- 9 files changed, 93 insertions(+), 111 deletions(-)

diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c index 70089d3..4e7dd2b 100644 --- a/drivers/gpu/drm/radeon/evergreen_cs.c +++ b/drivers/gpu/drm/radeon/evergreen_cs.c @@ -1057,7 +1057,7 @@ static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p) uint32_t header, h_idx, reg, wait_reg_mem_info; volatile uint32_t *ib;

- ib = p->ib->ptr; + ib = p->ib.ptr;

/* parse the WAIT_REG_MEM */ r = evergreen_cs_packet_parse(p, &wait_reg_mem, p->idx); @@ -1215,7 +1215,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) if (!(evergreen_reg_safe_bm[i] & m)) return 0; } - ib = p->ib->ptr; + ib = p->ib.ptr; switch (reg) { /* force following reg to 0 in an attempt to disable out buffer * which will need us to better understand how it works to perform @@ -1896,7 +1896,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p, u32 idx_value;

track = (struct evergreen_cs_track *)p->track; - ib = p->ib->ptr; + ib = p->ib.ptr; idx = pkt->idx + 1; idx_value = radeon_get_ib_value(p, idx);

@@ -2610,8 +2610,8 @@ int evergreen_cs_parse(struct radeon_cs_parser *p) } } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); #if 0 - for (r = 0; r < p->ib->length_dw; r++) { - printk(KERN_INFO "%05d 0x%08X\n", r, p->ib->ptr[r]); + for (r = 0; r < p->ib.length_dw; r++) { + printk(KERN_INFO "%05d 0x%08X\n", r, p->ib.ptr[r]); mdelay(1); } #endif diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c index ad6ceb7..0874a6d 100644 --- a/drivers/gpu/drm/radeon/r100.c +++ b/drivers/gpu/drm/radeon/r100.c @@ -139,9 +139,9 @@ int r100_reloc_pitch_offset(struct radeon_cs_parser *p, }

tmp |= tile_flags; - p->ib->ptr[idx] = (value & 0x3fc00000) | tmp; + p->ib.ptr[idx] = (value & 0x3fc00000) | tmp; } else - p->ib->ptr[idx] = (value & 0xffc00000) | tmp; + p->ib.ptr[idx] = (value & 0xffc00000) | tmp; return 0; }

@@ -156,7 +156,7 @@ int r100_packet3_load_vbpntr(struct radeon_cs_parser *p, volatile uint32_t *ib; u32 idx_value;

- ib = p->ib->ptr; + ib = p->ib.ptr; track = (struct r100_cs_track *)p->track; c = radeon_get_ib_value(p, idx++) & 0x1F; if (c > 16) { @@ -1275,7 +1275,7 @@ void r100_cs_dump_packet(struct radeon_cs_parser *p, unsigned i; unsigned idx;

- ib = p->ib->ptr; + ib = p->ib.ptr; idx = pkt->idx; for (i = 0; i <= (pkt->count + 1); i++, idx++) { DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]); @@ -1354,7 +1354,7 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p) uint32_t header, h_idx, reg; volatile uint32_t *ib;

- ib = p->ib->ptr; + ib = p->ib.ptr;

/* parse the wait until */ r = r100_cs_packet_parse(p, &waitreloc, p->idx); @@ -1533,7 +1533,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p, u32 tile_flags = 0; u32 idx_value;

- ib = p->ib->ptr; + ib = p->ib.ptr; track = (struct r100_cs_track *)p->track;

idx_value = radeon_get_ib_value(p, idx); @@ -1889,7 +1889,7 @@ static int r100_packet3_check(struct radeon_cs_parser *p, volatile uint32_t *ib; int r;

- ib = p->ib->ptr; + ib = p->ib.ptr; idx = pkt->idx + 1; track = (struct r100_cs_track *)p->track; switch (pkt->opcode) { @@ -3684,7 +3684,7 @@ void r100_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)

int r100_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) { - struct radeon_ib *ib; + struct radeon_ib ib; uint32_t scratch; uint32_t tmp = 0; unsigned i; @@ -3700,22 +3700,22 @@ int r100_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) if (r) { return r; } - ib->ptr[0] = PACKET0(scratch, 0); - ib->ptr[1] = 0xDEADBEEF; - ib->ptr[2] = PACKET2(0); - ib->ptr[3] = PACKET2(0); - ib->ptr[4] = PACKET2(0); - ib->ptr[5] = PACKET2(0); - ib->ptr[6] = PACKET2(0); - ib->ptr[7] = PACKET2(0); - ib->length_dw = 8; - r = radeon_ib_schedule(rdev, ib); + ib.ptr[0] = PACKET0(scratch, 0); + ib.ptr[1] = 0xDEADBEEF; + ib.ptr[2] = PACKET2(0); + ib.ptr[3] = PACKET2(0); + ib.ptr[4] = PACKET2(0); + ib.ptr[5] = PACKET2(0); + ib.ptr[6] = PACKET2(0); + ib.ptr[7] = PACKET2(0); + ib.length_dw = 8; + r = radeon_ib_schedule(rdev, &ib); if (r) { radeon_scratch_free(rdev, scratch); radeon_ib_free(rdev, &ib); return r; } - r = radeon_fence_wait(ib->fence, false); + r = radeon_fence_wait(ib.fence, false); if (r) { return r; } diff --git a/drivers/gpu/drm/radeon/r200.c b/drivers/gpu/drm/radeon/r200.c index a59cc47..a26144d 100644 --- a/drivers/gpu/drm/radeon/r200.c +++ b/drivers/gpu/drm/radeon/r200.c @@ -154,7 +154,7 @@ int r200_packet0_check(struct radeon_cs_parser *p, u32 tile_flags = 0; u32 idx_value;

- ib = p->ib->ptr; + ib = p->ib.ptr; track = (struct r100_cs_track *)p->track; idx_value = radeon_get_ib_value(p, idx); switch (reg) { diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c index 6419a59..97722a3 100644 --- a/drivers/gpu/drm/radeon/r300.c +++ b/drivers/gpu/drm/radeon/r300.c @@ -604,7 +604,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p, int r; u32 idx_value;

- ib = p->ib->ptr; + ib = p->ib.ptr; track = (struct r100_cs_track *)p->track; idx_value = radeon_get_ib_value(p, idx);

@@ -1146,7 +1146,7 @@ static int r300_packet3_check(struct radeon_cs_parser *p, unsigned idx; int r;

- ib = p->ib->ptr; + ib = p->ib.ptr; idx = pkt->idx + 1; track = (struct r100_cs_track *)p->track; switch(pkt->opcode) { diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index 00b2238..4c0d8c9 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -2681,7 +2681,7 @@ void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)

int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) { - struct radeon_ib *ib; + struct radeon_ib ib; uint32_t scratch; uint32_t tmp = 0; unsigned i; @@ -2699,18 +2699,18 @@ int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) DRM_ERROR("radeon: failed to get ib (%d).\n", r); return r; } - ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1); - ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); - ib->ptr[2] = 0xDEADBEEF; - ib->length_dw = 3; - r = radeon_ib_schedule(rdev, ib); + ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1); + ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); + ib.ptr[2] = 0xDEADBEEF; + ib.length_dw = 3; + r = radeon_ib_schedule(rdev, &ib); if (r) { radeon_scratch_free(rdev, scratch); radeon_ib_free(rdev, &ib); DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); return r; } - r = radeon_fence_wait(ib->fence, false); + r = radeon_fence_wait(ib.fence, false); if (r) { DRM_ERROR("radeon: fence wait failed (%d).\n", r); return r; @@ -2722,7 +2722,7 @@ int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) DRM_UDELAY(1); } if (i < rdev->usec_timeout) { - DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib->fence->ring, i); + DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); } else { DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp); diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c index b8e12af..0133f5f 100644 --- a/drivers/gpu/drm/radeon/r600_cs.c +++ b/drivers/gpu/drm/radeon/r600_cs.c @@ -345,7 +345,7 @@ static int r600_cs_track_validate_cb(struct radeon_cs_parser *p, int i) u32 height, height_align, pitch, pitch_align, depth_align; u64 base_offset, base_align; struct array_mode_checker array_check; - volatile u32 *ib = p->ib->ptr; + volatile u32 *ib = p->ib.ptr; unsigned array_mode; u32 format;

@@ -471,7 +471,7 @@ static int r600_cs_track_validate_db(struct radeon_cs_parser *p) u64 base_offset, base_align; struct array_mode_checker array_check; int array_mode; - volatile u32 *ib = p->ib->ptr; + volatile u32 *ib = p->ib.ptr;

if (track->db_bo == NULL) { @@ -961,7 +961,7 @@ static int r600_cs_packet_parse_vline(struct radeon_cs_parser *p) uint32_t header, h_idx, reg, wait_reg_mem_info; volatile uint32_t *ib;

- ib = p->ib->ptr; + ib = p->ib.ptr;

/* parse the WAIT_REG_MEM */ r = r600_cs_packet_parse(p, &wait_reg_mem, p->idx); @@ -1110,7 +1110,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) m = 1 << ((reg >> 2) & 31); if (!(r600_reg_safe_bm[i] & m)) return 0; - ib = p->ib->ptr; + ib = p->ib.ptr; switch (reg) { /* force following reg to 0 in an attempt to disable out buffer * which will need us to better understand how it works to perform @@ -1714,7 +1714,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p, u32 idx_value;

track = (struct r600_cs_track *)p->track; - ib = p->ib->ptr; + ib = p->ib.ptr; idx = pkt->idx + 1; idx_value = radeon_get_ib_value(p, idx);

@@ -2249,8 +2249,8 @@ int r600_cs_parse(struct radeon_cs_parser *p) } } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw); #if 0 - for (r = 0; r < p->ib->length_dw; r++) { - printk(KERN_INFO "%05d 0x%08X\n", r, p->ib->ptr[r]); + for (r = 0; r < p->ib.length_dw; r++) { + printk(KERN_INFO "%05d 0x%08X\n", r, p->ib.ptr[r]); mdelay(1); } #endif @@ -2298,7 +2298,6 @@ int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp, { struct radeon_cs_parser parser; struct radeon_cs_chunk *ib_chunk; - struct radeon_ib fake_ib; struct r600_cs_track *track; int r;

@@ -2314,9 +2313,8 @@ int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp, parser.dev = &dev->pdev->dev; parser.rdev = NULL; parser.family = family; - parser.ib = &fake_ib; parser.track = track; - fake_ib.ptr = ib; + parser.ib.ptr = ib; r = radeon_cs_parser_init(&parser, data); if (r) { DRM_ERROR("Failed to initialize parser !\n"); @@ -2333,8 +2331,8 @@ int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp, * input memory (cached) and write to the IB (which can be * uncached). */ ib_chunk = &parser.chunks[parser.chunk_ib_idx]; - parser.ib->length_dw = ib_chunk->length_dw; - *l = parser.ib->length_dw; + parser.ib.length_dw = ib_chunk->length_dw; + *l = parser.ib.length_dw; r = r600_cs_parse(&parser); if (r) { DRM_ERROR("Invalid command stream !\n"); diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 659855a..60233d7 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -769,8 +769,8 @@ struct si_rlc { };

int radeon_ib_get(struct radeon_device *rdev, int ring, - struct radeon_ib **ib, unsigned size); -void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib); + struct radeon_ib *ib, unsigned size); +void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib); int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib); int radeon_ib_pool_init(struct radeon_device *rdev); void radeon_ib_pool_fini(struct radeon_device *rdev); @@ -838,8 +838,8 @@ struct radeon_cs_parser { int chunk_relocs_idx; int chunk_flags_idx; int chunk_const_ib_idx; - struct radeon_ib *ib; - struct radeon_ib *const_ib; + struct radeon_ib ib; + struct radeon_ib const_ib; void *track; unsigned family; int parser_error; diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index dcfe2a0..c7d64a7 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -138,12 +138,12 @@ static int radeon_cs_sync_rings(struct radeon_cs_parser *p) return 0; }

- r = radeon_semaphore_create(p->rdev, &p->ib->semaphore); + r = radeon_semaphore_create(p->rdev, &p->ib.semaphore); if (r) { return r; }

- return radeon_semaphore_sync_rings(p->rdev, p->ib->semaphore, + return radeon_semaphore_sync_rings(p->rdev, p->ib.semaphore, sync_to_ring, p->ring); }

@@ -161,8 +161,10 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) /* get chunks */ INIT_LIST_HEAD(&p->validated); p->idx = 0; - p->ib = NULL; - p->const_ib = NULL; + p->ib.sa_bo = NULL; + p->ib.semaphore = NULL; + p->const_ib.sa_bo = NULL; + p->const_ib.semaphore = NULL; p->chunk_ib_idx = -1; p->chunk_relocs_idx = -1; p->chunk_flags_idx = -1; @@ -301,10 +303,9 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error) { unsigned i;

- - if (!error && parser->ib) + if (!error) ttm_eu_fence_buffer_objects(&parser->validated, - parser->ib->fence); + parser->ib.fence); else ttm_eu_backoff_reservation(&parser->validated);

@@ -327,9 +328,7 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error) kfree(parser->chunks); kfree(parser->chunks_array); radeon_ib_free(parser->rdev, &parser->ib); - if (parser->const_ib) { - radeon_ib_free(parser->rdev, &parser->const_ib); - } + radeon_ib_free(parser->rdev, &parser->const_ib); }

static int radeon_cs_ib_chunk(struct radeon_device *rdev, @@ -355,7 +354,7 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev, DRM_ERROR("Failed to get ib !\n"); return r; } - parser->ib->length_dw = ib_chunk->length_dw; + parser->ib.length_dw = ib_chunk->length_dw; r = radeon_cs_parse(rdev, parser->ring, parser); if (r || parser->parser_error) { DRM_ERROR("Invalid command stream !\n"); @@ -370,8 +369,8 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev, if (r) { DRM_ERROR("Failed to synchronize rings !\n"); } - parser->ib->vm_id = 0; - r = radeon_ib_schedule(rdev, parser->ib); + parser->ib.vm_id = 0; + r = radeon_ib_schedule(rdev, &parser->ib); if (r) { DRM_ERROR("Failed to schedule IB !\n"); } @@ -422,14 +421,14 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, DRM_ERROR("Failed to get const ib !\n"); return r; } - parser->const_ib->is_const_ib = true; - parser->const_ib->length_dw = ib_chunk->length_dw; + parser->const_ib.is_const_ib = true; + parser->const_ib.length_dw = ib_chunk->length_dw; /* Copy the packet into the IB */ - if (DRM_COPY_FROM_USER(parser->const_ib->ptr, ib_chunk->user_ptr, + if (DRM_COPY_FROM_USER(parser->const_ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4)) { return -EFAULT; } - r = radeon_ring_ib_parse(rdev, parser->ring, parser->const_ib); + r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib); if (r) { return r; } @@ -446,13 +445,13 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, DRM_ERROR("Failed to get ib !\n"); return r; } - parser->ib->length_dw = ib_chunk->length_dw; + parser->ib.length_dw = ib_chunk->length_dw; /* Copy the packet into the IB */ - if (DRM_COPY_FROM_USER(parser->ib->ptr, ib_chunk->user_ptr, + if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4)) { return -EFAULT; } - r = radeon_ring_ib_parse(rdev, parser->ring, parser->ib); + r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib); if (r) { return r; } @@ -473,29 +472,29 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,

if ((rdev->family >= CHIP_TAHITI) && (parser->chunk_const_ib_idx != -1)) { - parser->const_ib->vm_id = vm->id; + parser->const_ib.vm_id = vm->id; /* ib pool is bind at 0 in virtual address space to gpu_addr is the * offset inside the pool bo */ - parser->const_ib->gpu_addr = parser->const_ib->sa_bo->soffset; - r = radeon_ib_schedule(rdev, parser->const_ib); + parser->const_ib.gpu_addr = parser->const_ib.sa_bo->soffset; + r = radeon_ib_schedule(rdev, &parser->const_ib); if (r) goto out; }

- parser->ib->vm_id = vm->id; + parser->ib.vm_id = vm->id; /* ib pool is bind at 0 in virtual address space to gpu_addr is the * offset inside the pool bo */ - parser->ib->gpu_addr = parser->ib->sa_bo->soffset; - parser->ib->is_const_ib = false; - r = radeon_ib_schedule(rdev, parser->ib); + parser->ib.gpu_addr = parser->ib.sa_bo->soffset; + parser->ib.is_const_ib = false; + r = radeon_ib_schedule(rdev, &parser->ib); out: if (!r) { if (vm->fence) { radeon_fence_unref(&vm->fence); } - vm->fence = radeon_fence_ref(parser->ib->fence); + vm->fence = radeon_fence_ref(parser->ib.fence); } mutex_unlock(&fpriv->vm.mutex); return r; @@ -573,7 +572,7 @@ int radeon_cs_finish_pages(struct radeon_cs_parser *p) size = PAGE_SIZE; } - if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)), + if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)), ibc->user_ptr + (i * PAGE_SIZE), size)) return -EFAULT; @@ -590,7 +589,7 @@ int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx) bool copy1 = (p->rdev->flags & RADEON_IS_AGP) ? false : true;

for (i = ibc->last_copied_page + 1; i < pg_idx; i++) { - if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)), + if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)), ibc->user_ptr + (i * PAGE_SIZE), PAGE_SIZE)) { p->parser_error = -EFAULT; @@ -606,7 +605,7 @@ int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)

new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1; if (copy1) - ibc->kpage[new_page] = p->ib->ptr + (pg_idx * (PAGE_SIZE / 4)); + ibc->kpage[new_page] = p->ib.ptr + (pg_idx * (PAGE_SIZE / 4));

if (DRM_COPY_FROM_USER(ibc->kpage[new_page], ibc->user_ptr + (pg_idx * PAGE_SIZE), @@ -617,7 +616,7 @@ int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)

/* copy to IB for non single case */ if (!copy1) - memcpy((void *)(p->ib->ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size); + memcpy((void *)(p->ib.ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size);

ibc->last_copied_page = pg_idx; ibc->kpage_idx[new_page] = pg_idx; diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index af8e1ee..a5dee76 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -65,51 +65,36 @@ u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx) }

int radeon_ib_get(struct radeon_device *rdev, int ring, - struct radeon_ib **ib, unsigned size) + struct radeon_ib *ib, unsigned size) { int r;

- *ib = kmalloc(sizeof(struct radeon_ib), GFP_KERNEL); - if (*ib == NULL) { - return -ENOMEM; - } - r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &(*ib)->sa_bo, size, 256, true); + r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true); if (r) { dev_err(rdev->dev, "failed to get a new IB (%d)\n", r); - kfree(*ib); - *ib = NULL; return r; } - r = radeon_fence_create(rdev, &(*ib)->fence, ring); + r = radeon_fence_create(rdev, &ib->fence, ring); if (r) { dev_err(rdev->dev, "failed to create fence for new IB (%d)\n", r); - radeon_sa_bo_free(rdev, &(*ib)->sa_bo, NULL); - kfree(*ib); - *ib = NULL; + radeon_sa_bo_free(rdev, &ib->sa_bo, NULL); return r; }

- (*ib)->ptr = radeon_sa_bo_cpu_addr((*ib)->sa_bo); - (*ib)->gpu_addr = radeon_sa_bo_gpu_addr((*ib)->sa_bo); - (*ib)->vm_id = 0; - (*ib)->is_const_ib = false; - (*ib)->semaphore = NULL; + ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo); + ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo); + ib->vm_id = 0; + ib->is_const_ib = false; + ib->semaphore = NULL;

return 0; }

-void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib) +void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib) { - struct radeon_ib *tmp = *ib; - - *ib = NULL; - if (tmp == NULL) { - return; - } - radeon_semaphore_free(rdev, tmp->semaphore, tmp->fence); - radeon_sa_bo_free(rdev, &tmp->sa_bo, tmp->fence); - radeon_fence_unref(&tmp->fence); - kfree(tmp); + radeon_semaphore_free(rdev, ib->semaphore, ib->fence); + radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence); + radeon_fence_unref(&ib->fence); }

int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)