When writeback is enabled, the GPU shadows writes to certain registers into a buffer in memory. The driver can then read the values from the shadow rather than reading back from the register. Writeback can be disabled by setting the no_wb module param to 0.
On r6xx/r7xx/evergreen, the following registers are shadowed: - CP scratch registers - CP read pointer - IH write pointer
Signed-off-by: Alex Deucher alexdeucher@gmail.com --- drivers/gpu/drm/radeon/evergreen.c | 36 +++++++--- drivers/gpu/drm/radeon/r600.c | 116 ++++++++++---------------------- drivers/gpu/drm/radeon/radeon.h | 12 +++- drivers/gpu/drm/radeon/radeon_device.c | 72 +++++++++++++++++++- drivers/gpu/drm/radeon/radeon_fence.c | 6 ++- drivers/gpu/drm/radeon/radeon_ring.c | 12 ++- drivers/gpu/drm/radeon/rv770.c | 15 +++-- 7 files changed, 166 insertions(+), 103 deletions(-)
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c index 957d506..c341187 100644 --- a/drivers/gpu/drm/radeon/evergreen.c +++ b/drivers/gpu/drm/radeon/evergreen.c @@ -694,7 +694,7 @@ int evergreen_cp_resume(struct radeon_device *rdev)
/* Set ring buffer size */ rb_bufsz = drm_order(rdev->cp.ring_size / 8); - tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -708,8 +708,19 @@ int evergreen_cp_resume(struct radeon_device *rdev) WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); WREG32(CP_RB_RPTR_WR, 0); WREG32(CP_RB_WPTR, 0); - WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF); - WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr)); + + /* set the wb address wether it's enabled or not */ + WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); + WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); + WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); + + if (rdev->wb.enabled) + WREG32(SCRATCH_UMSK, 0xff); + else { + tmp |= RB_NO_UPDATE; + WREG32(SCRATCH_UMSK, 0); + } + mdelay(1); WREG32(CP_RB_CNTL, tmp);
@@ -1711,8 +1722,10 @@ static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev) { u32 wptr, tmp;
- /* XXX use writeback */ - wptr = RREG32(IH_RB_WPTR); + if (rdev->wb.enabled) + wptr = rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]; + else + wptr = RREG32(IH_RB_WPTR);
if (wptr & RB_OVERFLOW) { /* When a ring buffer overflow happen start parsing interrupt @@ -2020,6 +2033,11 @@ static int evergreen_startup(struct radeon_device *rdev) } #endif
+ /* allocate wb buffer */ + radeon_wb_init(rdev); + if (r) + return r; + /* Enable IRQ */ r = r600_irq_init(rdev); if (r) { @@ -2038,8 +2056,6 @@ static int evergreen_startup(struct radeon_device *rdev) r = evergreen_cp_resume(rdev); if (r) return r; - /* write back buffer are not vital so don't worry about failure */ - r600_wb_enable(rdev);
return 0; } @@ -2085,7 +2101,7 @@ int evergreen_suspend(struct radeon_device *rdev) r700_cp_stop(rdev); rdev->cp.ready = false; evergreen_irq_suspend(rdev); - r600_wb_disable(rdev); + radeon_wb_disable(rdev); evergreen_pcie_gart_disable(rdev); #if 0 /* unpin shaders bo */ @@ -2205,8 +2221,8 @@ int evergreen_init(struct radeon_device *rdev) if (r) { dev_err(rdev->dev, "disabling GPU acceleration\n"); r700_cp_fini(rdev); - r600_wb_fini(rdev); r600_irq_fini(rdev); + radeon_wb_fini(rdev); radeon_irq_kms_fini(rdev); evergreen_pcie_gart_fini(rdev); rdev->accel_working = false; @@ -2230,8 +2246,8 @@ void evergreen_fini(struct radeon_device *rdev) { /*r600_blit_fini(rdev);*/ r700_cp_fini(rdev); - r600_wb_fini(rdev); r600_irq_fini(rdev); + radeon_wb_fini(rdev); radeon_irq_kms_fini(rdev); evergreen_pcie_gart_fini(rdev); radeon_gem_fini(rdev); diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index cc8544f..cb6319d 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -1918,6 +1918,7 @@ void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) void r600_cp_stop(struct radeon_device *rdev) { WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1)); + WREG32(SCRATCH_UMSK, 0); }
int r600_init_microcode(struct radeon_device *rdev) @@ -2153,7 +2154,7 @@ int r600_cp_resume(struct radeon_device *rdev)
/* Set ring buffer size */ rb_bufsz = drm_order(rdev->cp.ring_size / 8); - tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -2167,8 +2168,19 @@ int r600_cp_resume(struct radeon_device *rdev) WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); WREG32(CP_RB_RPTR_WR, 0); WREG32(CP_RB_WPTR, 0); - WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF); - WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr)); + + /* set the wb address wether it's enabled or not */ + WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); + WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); + WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); + + if (rdev->wb.enabled) + WREG32(SCRATCH_UMSK, 0xff); + else { + tmp |= RB_NO_UPDATE; + WREG32(SCRATCH_UMSK, 0); + } + mdelay(1); WREG32(CP_RB_CNTL, tmp);
@@ -2220,9 +2232,10 @@ void r600_scratch_init(struct radeon_device *rdev) int i;
rdev->scratch.num_reg = 7; + rdev->scratch.reg_base = SCRATCH_REG0; for (i = 0; i < rdev->scratch.num_reg; i++) { rdev->scratch.free[i] = true; - rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4); + rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); } }
@@ -2266,70 +2279,6 @@ int r600_ring_test(struct radeon_device *rdev) return r; }
-void r600_wb_disable(struct radeon_device *rdev) -{ - int r; - - WREG32(SCRATCH_UMSK, 0); - if (rdev->wb.wb_obj) { - r = radeon_bo_reserve(rdev->wb.wb_obj, false); - if (unlikely(r != 0)) - return; - radeon_bo_kunmap(rdev->wb.wb_obj); - radeon_bo_unpin(rdev->wb.wb_obj); - radeon_bo_unreserve(rdev->wb.wb_obj); - } -} - -void r600_wb_fini(struct radeon_device *rdev) -{ - r600_wb_disable(rdev); - if (rdev->wb.wb_obj) { - radeon_bo_unref(&rdev->wb.wb_obj); - rdev->wb.wb = NULL; - rdev->wb.wb_obj = NULL; - } -} - -int r600_wb_enable(struct radeon_device *rdev) -{ - int r; - - if (rdev->wb.wb_obj == NULL) { - r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true, - RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj); - if (r) { - dev_warn(rdev->dev, "(%d) create WB bo failed\n", r); - return r; - } - r = radeon_bo_reserve(rdev->wb.wb_obj, false); - if (unlikely(r != 0)) { - r600_wb_fini(rdev); - return r; - } - r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT, - &rdev->wb.gpu_addr); - if (r) { - radeon_bo_unreserve(rdev->wb.wb_obj); - dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r); - r600_wb_fini(rdev); - return r; - } - r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb); - radeon_bo_unreserve(rdev->wb.wb_obj); - if (r) { - dev_warn(rdev->dev, "(%d) map WB bo failed\n", r); - r600_wb_fini(rdev); - return r; - } - } - WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF); - WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC); - WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF); - WREG32(SCRATCH_UMSK, 0xff); - return 0; -} - void r600_fence_ring_emit(struct radeon_device *rdev, struct radeon_fence *fence) { @@ -2430,6 +2379,11 @@ int r600_startup(struct radeon_device *rdev) dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); }
+ /* allocate wb buffer */ + radeon_wb_init(rdev); + if (r) + return r; + /* Enable IRQ */ r = r600_irq_init(rdev); if (r) { @@ -2448,8 +2402,7 @@ int r600_startup(struct radeon_device *rdev) r = r600_cp_resume(rdev); if (r) return r; - /* write back buffer are not vital so don't worry about failure */ - r600_wb_enable(rdev); + return 0; }
@@ -2513,7 +2466,7 @@ int r600_suspend(struct radeon_device *rdev) r600_cp_stop(rdev); rdev->cp.ready = false; r600_irq_suspend(rdev); - r600_wb_disable(rdev); + radeon_wb_disable(rdev); r600_pcie_gart_disable(rdev); /* unpin shaders bo */ if (rdev->r600_blit.shader_obj) { @@ -2613,8 +2566,8 @@ int r600_init(struct radeon_device *rdev) if (r) { dev_err(rdev->dev, "disabling GPU acceleration\n"); r600_cp_fini(rdev); - r600_wb_fini(rdev); r600_irq_fini(rdev); + radeon_wb_fini(rdev); radeon_irq_kms_fini(rdev); r600_pcie_gart_fini(rdev); rdev->accel_working = false; @@ -2644,8 +2597,8 @@ void r600_fini(struct radeon_device *rdev) r600_audio_fini(rdev); r600_blit_fini(rdev); r600_cp_fini(rdev); - r600_wb_fini(rdev); r600_irq_fini(rdev); + radeon_wb_fini(rdev); radeon_irq_kms_fini(rdev); r600_pcie_gart_fini(rdev); radeon_agp_fini(rdev); @@ -2981,10 +2934,13 @@ int r600_irq_init(struct radeon_device *rdev) ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | IH_WPTR_OVERFLOW_CLEAR | (rb_bufsz << 1)); - /* WPTR writeback, not yet */ - /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/ - WREG32(IH_RB_WPTR_ADDR_LO, 0); - WREG32(IH_RB_WPTR_ADDR_HI, 0); + + if (rdev->wb.enabled) + ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; + + /* set the writeback address whether it's enabled or not */ + WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); + WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
WREG32(IH_RB_CNTL, ih_rb_cntl);
@@ -3242,8 +3198,10 @@ static inline u32 r600_get_ih_wptr(struct radeon_device *rdev) { u32 wptr, tmp;
- /* XXX use writeback */ - wptr = RREG32(IH_RB_WPTR); + if (rdev->wb.enabled) + wptr = rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]; + else + wptr = RREG32(IH_RB_WPTR);
if (wptr & RB_OVERFLOW) { /* When a ring buffer overflow happen start parsing interrupt diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index fed3d07..0daa039 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -365,6 +365,7 @@ bool radeon_atombios_sideport_present(struct radeon_device *rdev); */ struct radeon_scratch { unsigned num_reg; + uint32_t reg_base; bool free[32]; uint32_t reg[32]; }; @@ -593,8 +594,13 @@ struct radeon_wb { struct radeon_bo *wb_obj; volatile uint32_t *wb; uint64_t gpu_addr; + bool enabled; };
+#define RADEON_WB_SCRATCH_OFFSET 0 +#define RADEON_WB_CP_RPTR_OFFSET 1024 +#define R600_WB_IH_WPTR_OFFSET 2048 + /** * struct radeon_pm - power management datas * @max_bandwidth: maximum bandwidth the gpu has (MByte/s) @@ -1336,6 +1342,9 @@ extern bool radeon_boot_test_post_card(struct radeon_device *rdev); extern int radeon_clocks_init(struct radeon_device *rdev); extern void radeon_clocks_fini(struct radeon_device *rdev); extern void radeon_scratch_init(struct radeon_device *rdev); +extern void radeon_wb_fini(struct radeon_device *rdev); +extern int radeon_wb_init(struct radeon_device *rdev); +extern void radeon_wb_disable(struct radeon_device *rdev); extern void radeon_surface_init(struct radeon_device *rdev); extern int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data); extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable); @@ -1420,9 +1429,6 @@ extern int r600_pcie_gart_init(struct radeon_device *rdev); extern void r600_pcie_gart_tlb_flush(struct radeon_device *rdev); extern int r600_ib_test(struct radeon_device *rdev); extern int r600_ring_test(struct radeon_device *rdev); -extern void r600_wb_fini(struct radeon_device *rdev); -extern int r600_wb_enable(struct radeon_device *rdev); -extern void r600_wb_disable(struct radeon_device *rdev); extern void r600_scratch_init(struct radeon_device *rdev); extern int r600_blit_init(struct radeon_device *rdev); extern void r600_blit_fini(struct radeon_device *rdev); diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index a64811a..39a2b97 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -117,9 +117,10 @@ void radeon_scratch_init(struct radeon_device *rdev) } else { rdev->scratch.num_reg = 7; } + rdev->scratch.reg_base = RADEON_SCRATCH_REG0; for (i = 0; i < rdev->scratch.num_reg; i++) { rdev->scratch.free[i] = true; - rdev->scratch.reg[i] = RADEON_SCRATCH_REG0 + (i * 4); + rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); } }
@@ -149,6 +150,75 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg) } }
+void radeon_wb_disable(struct radeon_device *rdev) +{ + int r; + + if (rdev->wb.wb_obj) { + r = radeon_bo_reserve(rdev->wb.wb_obj, false); + if (unlikely(r != 0)) + return; + radeon_bo_kunmap(rdev->wb.wb_obj); + radeon_bo_unpin(rdev->wb.wb_obj); + radeon_bo_unreserve(rdev->wb.wb_obj); + } + rdev->wb.enabled = false; +} + +void radeon_wb_fini(struct radeon_device *rdev) +{ + radeon_wb_disable(rdev); + if (rdev->wb.wb_obj) { + radeon_bo_unref(&rdev->wb.wb_obj); + rdev->wb.wb = NULL; + rdev->wb.wb_obj = NULL; + } +} + +int radeon_wb_init(struct radeon_device *rdev) +{ + int r; + + if (rdev->wb.wb_obj == NULL) { + r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true, + RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create WB bo failed\n", r); + return r; + } + } + r = radeon_bo_reserve(rdev->wb.wb_obj, false); + if (unlikely(r != 0)) { + radeon_wb_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT, + &rdev->wb.gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->wb.wb_obj); + dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r); + radeon_wb_fini(rdev); + return r; + } + r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb); + radeon_bo_unreserve(rdev->wb.wb_obj); + if (r) { + dev_warn(rdev->dev, "(%d) map WB bo failed\n", r); + radeon_wb_fini(rdev); + return r; + } + + /* disabled via module param */ + if (radeon_no_wb == 1) + rdev->wb.enabled = false; + else + rdev->wb.enabled = true; + + dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis"); + + return 0; +} + /** * radeon_vram_location - try to find VRAM location * @rdev: radeon device structure holding all necessary informations diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index b1f9a81..698a7ed 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -72,7 +72,11 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev) bool wake = false; unsigned long cjiffies;
- seq = RREG32(rdev->fence_drv.scratch_reg); + if (rdev->wb.enabled) { + u32 scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; + seq = rdev->wb.wb[scratch_index/4]; + } else + seq = RREG32(rdev->fence_drv.scratch_reg); if (seq != rdev->fence_drv.last_seq) { rdev->fence_drv.last_seq = seq; rdev->fence_drv.last_jiffies = jiffies; diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index 261e98a..6ea798c 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -247,10 +247,14 @@ void radeon_ib_pool_fini(struct radeon_device *rdev) */ void radeon_ring_free_size(struct radeon_device *rdev) { - if (rdev->family >= CHIP_R600) - rdev->cp.rptr = RREG32(R600_CP_RB_RPTR); - else - rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); + if (rdev->wb.enabled) + rdev->cp.rptr = rdev->wb.wb[RADEON_WB_CP_RPTR_OFFSET/4]; + else { + if (rdev->family >= CHIP_R600) + rdev->cp.rptr = RREG32(R600_CP_RB_RPTR); + else + rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); + } /* This works because ring_size is a power of 2 */ rdev->cp.ring_free_dw = (rdev->cp.rptr + (rdev->cp.ring_size / 4)); rdev->cp.ring_free_dw -= rdev->cp.wptr; diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c index a2635c7..c046115 100644 --- a/drivers/gpu/drm/radeon/rv770.c +++ b/drivers/gpu/drm/radeon/rv770.c @@ -268,6 +268,7 @@ static void rv770_mc_program(struct radeon_device *rdev) void r700_cp_stop(struct radeon_device *rdev) { WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT)); + WREG32(SCRATCH_UMSK, 0); }
static int rv770_cp_load_microcode(struct radeon_device *rdev) @@ -978,6 +979,11 @@ static int rv770_startup(struct radeon_device *rdev) dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); }
+ /* allocate wb buffer */ + radeon_wb_init(rdev); + if (r) + return r; + /* Enable IRQ */ r = r600_irq_init(rdev); if (r) { @@ -996,8 +1002,7 @@ static int rv770_startup(struct radeon_device *rdev) r = r600_cp_resume(rdev); if (r) return r; - /* write back buffer are not vital so don't worry about failure */ - r600_wb_enable(rdev); + return 0; }
@@ -1048,7 +1053,7 @@ int rv770_suspend(struct radeon_device *rdev) r700_cp_stop(rdev); rdev->cp.ready = false; r600_irq_suspend(rdev); - r600_wb_disable(rdev); + radeon_wb_disable(rdev); rv770_pcie_gart_disable(rdev); /* unpin shaders bo */ if (rdev->r600_blit.shader_obj) { @@ -1146,8 +1151,8 @@ int rv770_init(struct radeon_device *rdev) if (r) { dev_err(rdev->dev, "disabling GPU acceleration\n"); r700_cp_fini(rdev); - r600_wb_fini(rdev); r600_irq_fini(rdev); + radeon_wb_fini(rdev); radeon_irq_kms_fini(rdev); rv770_pcie_gart_fini(rdev); rdev->accel_working = false; @@ -1179,8 +1184,8 @@ void rv770_fini(struct radeon_device *rdev) { r600_blit_fini(rdev); r700_cp_fini(rdev); - r600_wb_fini(rdev); r600_irq_fini(rdev); + radeon_wb_fini(rdev); radeon_irq_kms_fini(rdev); rv770_pcie_gart_fini(rdev); radeon_gem_fini(rdev);