From: Jerome Glisse jglisse@redhat.com
This convert fence to use uint64_t sequence number intention is to use the fact that uin64_t is big enough that we don't need to care about wrap around.
Tested with and without writeback using 0xFFFFF000 as initial fence sequence and thus allowing to test the wrap around from 32bits to 64bits.
Signed-off-by: Jerome Glisse jglisse@redhat.com --- drivers/gpu/drm/radeon/ni.c | 6 ++-- drivers/gpu/drm/radeon/r100.c | 5 ++- drivers/gpu/drm/radeon/r300.c | 5 ++- drivers/gpu/drm/radeon/r600.c | 11 ++++---- drivers/gpu/drm/radeon/radeon.h | 20 +++++++------- drivers/gpu/drm/radeon/radeon_fence.c | 43 +++++++++++++++++---------------- drivers/gpu/drm/radeon/si.c | 6 ++-- 7 files changed, 50 insertions(+), 46 deletions(-)
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c index 3160a74..416d7c2 100644 --- a/drivers/gpu/drm/radeon/ni.c +++ b/drivers/gpu/drm/radeon/ni.c @@ -1120,9 +1120,9 @@ void cayman_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5)); radeon_ring_write(ring, addr & 0xffffffff); - radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); - radeon_ring_write(ring, fence->seq); - radeon_ring_write(ring, 0); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(2) | INT_SEL(2)); + radeon_ring_write(ring, lower_32_bits(fence->seq)); + radeon_ring_write(ring, upper_32_bits(fence->seq)); }
void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c index 80b57c5..d6bd9ea 100644 --- a/drivers/gpu/drm/radeon/r100.c +++ b/drivers/gpu/drm/radeon/r100.c @@ -864,8 +864,9 @@ void r100_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0)); radeon_ring_write(ring, rdev->config.r100.hdp_cntl); /* Emit fence sequence & fire IRQ */ - radeon_ring_write(ring, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 0)); - radeon_ring_write(ring, fence->seq); + radeon_ring_write(ring, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 1)); + radeon_ring_write(ring, lower_32_bits(fence->seq)); + radeon_ring_write(ring, upper_32_bits(fence->seq)); radeon_ring_write(ring, PACKET0(RADEON_GEN_INT_STATUS, 0)); radeon_ring_write(ring, RADEON_SW_INT_FIRE); } diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c index 6419a59..7c9e30d 100644 --- a/drivers/gpu/drm/radeon/r300.c +++ b/drivers/gpu/drm/radeon/r300.c @@ -200,8 +200,9 @@ void r300_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0)); radeon_ring_write(ring, rdev->config.r300.hdp_cntl); /* Emit fence sequence & fire IRQ */ - radeon_ring_write(ring, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 0)); - radeon_ring_write(ring, fence->seq); + radeon_ring_write(ring, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 1)); + radeon_ring_write(ring, lower_32_bits(fence->seq)); + radeon_ring_write(ring, upper_32_bits(fence->seq)); radeon_ring_write(ring, PACKET0(RADEON_GEN_INT_STATUS, 0)); radeon_ring_write(ring, RADEON_SW_INT_FIRE); } diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index 02abf32..4fbc590 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -2313,9 +2313,9 @@ void r600_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5)); radeon_ring_write(ring, addr & 0xffffffff); - radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); - radeon_ring_write(ring, fence->seq); - radeon_ring_write(ring, 0); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(2) | INT_SEL(2)); + radeon_ring_write(ring, lower_32_bits(fence->seq)); + radeon_ring_write(ring, upper_32_bits(fence->seq)); } else { /* flush read cache over gart */ radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); @@ -2332,9 +2332,10 @@ void r600_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit); /* Emit fence sequence & fire IRQ */ - radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); + radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 2)); radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2)); - radeon_ring_write(ring, fence->seq); + radeon_ring_write(ring, lower_32_bits(fence->seq)); + radeon_ring_write(ring, upper_32_bits(fence->seq)); /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */ radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0)); radeon_ring_write(ring, RB_INT_STAT); diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 4f21b68..4c5a667 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -281,9 +281,9 @@ void radeon_semaphore_free(struct radeon_device *rdev, struct radeon_fence_driver { uint32_t scratch_reg; uint64_t gpu_addr; - volatile uint32_t *cpu_addr; - atomic_t seq; - uint32_t last_seq; + volatile uint64_t *cpu_addr; + atomic64_t seq; + uint64_t last_seq; unsigned long last_activity; wait_queue_head_t queue; struct list_head emitted; @@ -296,7 +296,7 @@ struct radeon_fence { struct kref kref; struct list_head list; /* protected by radeon_fence.lock */ - uint32_t seq; + uint64_t seq; bool emitted; bool signaled; /* RB, DMA, etc. */ @@ -884,12 +884,12 @@ struct radeon_wb { bool use_event; };
-#define RADEON_WB_SCRATCH_OFFSET 0 -#define RADEON_WB_CP_RPTR_OFFSET 1024 -#define RADEON_WB_CP1_RPTR_OFFSET 1280 -#define RADEON_WB_CP2_RPTR_OFFSET 1536 -#define R600_WB_IH_WPTR_OFFSET 2048 -#define R600_WB_EVENT_OFFSET 3072 +#define RADEON_WB_SCRATCH_OFFSET 0 +#define RADEON_WB_CP_RPTR_OFFSET 1024 +#define RADEON_WB_CP1_RPTR_OFFSET 1280 +#define RADEON_WB_CP2_RPTR_OFFSET 1536 +#define R600_WB_IH_WPTR_OFFSET 2048 +#define R600_WB_EVENT_OFFSET 3072
/** * struct radeon_pm - power management datas diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 7733429..6da1535 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -40,23 +40,25 @@ #include "radeon.h" #include "radeon_trace.h"
-static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) +static void radeon_fence_write(struct radeon_device *rdev, u64 seq, int ring) { if (rdev->wb.enabled) { - *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq); + *rdev->fence_drv[ring].cpu_addr = cpu_to_le64(seq); } else { - WREG32(rdev->fence_drv[ring].scratch_reg, seq); + WREG32(rdev->fence_drv[ring].scratch_reg, lower_32_bits(seq)); + WREG32(rdev->fence_drv[ring].scratch_reg + 4, upper_32_bits(seq)); } }
-static u32 radeon_fence_read(struct radeon_device *rdev, int ring) +static u64 radeon_fence_read(struct radeon_device *rdev, int ring) { - u32 seq = 0; + u64 seq = 0;
if (rdev->wb.enabled) { - seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr); + seq = le64_to_cpu(*rdev->fence_drv[ring].cpu_addr); } else { seq = RREG32(rdev->fence_drv[ring].scratch_reg); + seq |= ((u64)RREG32(rdev->fence_drv[ring].scratch_reg + 4)) << 32ULL; } return seq; } @@ -70,7 +72,7 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; } - fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq); + fence->seq = atomic64_inc_return(&rdev->fence_drv[fence->ring].seq); radeon_fence_ring_emit(rdev, fence->ring, fence); trace_radeon_fence_emit(rdev->ddev, fence->seq); fence->emitted = true; @@ -87,7 +89,7 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring) { struct radeon_fence *fence; struct list_head *i, *n; - uint32_t seq; + uint64_t seq; bool wake = false;
seq = radeon_fence_read(rdev, ring); @@ -185,7 +187,7 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) { struct radeon_device *rdev; unsigned long irq_flags, timeout; - u32 seq; + u64 seq; int i, r; bool signaled;
@@ -252,8 +254,8 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) {
/* good news we believe it's a lockup */ - printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n", - fence->seq, seq); + dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n", + fence->seq, seq);
/* mark the ring as not ready any more */ rdev->ring[fence->ring].ready = false; @@ -371,12 +373,12 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) int r;
write_lock_irqsave(&rdev->fence_lock, irq_flags); - radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg, 1); + radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg, 2); if (rdev->wb.use_event) { rdev->fence_drv[ring].scratch_reg = 0; - index = R600_WB_EVENT_OFFSET + ring * 4; + index = R600_WB_EVENT_OFFSET + ring * 8; } else { - r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg, 1); + r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg, 2); if (r) { dev_err(rdev->dev, "fence failed to get scratch register\n"); write_unlock_irqrestore(&rdev->fence_lock, irq_flags); @@ -386,9 +388,9 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; } - rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; + rdev->fence_drv[ring].cpu_addr = (u64*)&rdev->wb.wb[index/4]; rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; - radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring); + radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].seq), ring); rdev->fence_drv[ring].initialized = true; DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n", ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); @@ -401,7 +403,7 @@ static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) rdev->fence_drv[ring].scratch_reg = -1; rdev->fence_drv[ring].cpu_addr = NULL; rdev->fence_drv[ring].gpu_addr = 0; - atomic_set(&rdev->fence_drv[ring].seq, 0); + atomic64_set(&rdev->fence_drv[ring].seq, 0); INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); init_waitqueue_head(&rdev->fence_drv[ring].queue); @@ -435,7 +437,7 @@ void radeon_fence_driver_fini(struct radeon_device *rdev) radeon_fence_wait_empty(rdev, ring); wake_up_all(&rdev->fence_drv[ring].queue); write_lock_irqsave(&rdev->fence_lock, irq_flags); - radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg, 1); + radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg, 2); write_unlock_irqrestore(&rdev->fence_lock, irq_flags); rdev->fence_drv[ring].initialized = false; } @@ -459,12 +461,11 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data) continue;
seq_printf(m, "--- ring %d ---\n", i); - seq_printf(m, "Last signaled fence 0x%08X\n", - radeon_fence_read(rdev, i)); + seq_printf(m, "Last signaled fence 0x%016llx\n", radeon_fence_read(rdev, i)); if (!list_empty(&rdev->fence_drv[i].emitted)) { fence = list_entry(rdev->fence_drv[i].emitted.prev, struct radeon_fence, list); - seq_printf(m, "Last emitted fence %p with 0x%08X\n", + seq_printf(m, "Last emitted fence %p with 0x%016llx\n", fence, fence->seq); } } diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c index eb49483..6de9610 100644 --- a/drivers/gpu/drm/radeon/si.c +++ b/drivers/gpu/drm/radeon/si.c @@ -1905,9 +1905,9 @@ void si_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5)); radeon_ring_write(ring, addr & 0xffffffff); - radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); - radeon_ring_write(ring, fence->seq); - radeon_ring_write(ring, 0); + radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(2) | INT_SEL(2)); + radeon_ring_write(ring, lower_32_bits(fence->seq)); + radeon_ring_write(ring, upper_32_bits(fence->seq)); }
/*