[PATCH 2/2] radeon: fix stencil miptree allocation of combined ZS buffers on EG and SI

This allows texturing with depth-stencil buffers directly without the copy to CB. The separate miptree description for stencil is added, because the stencil mipmap offsets are not really depth offsets/4 (at least for the texture units). --- radeon/radeon_surface.c | 197 +++++++++++++++++++++++++++++------------------ radeon/radeon_surface.h | 2 + 2 files changed, 123 insertions(+), 76 deletions(-)

diff --git a/radeon/radeon_surface.c b/radeon/radeon_surface.c index c62cbf9..66c2444 100644 --- a/radeon/radeon_surface.c +++ b/radeon/radeon_surface.c @@ -144,31 +144,32 @@ static unsigned mip_minify(unsigned size, unsigned level) }

static void surf_minify(struct radeon_surface *surf, - unsigned level, + struct radeon_surface_level *surflevel, + unsigned bpe, unsigned level, uint32_t xalign, uint32_t yalign, uint32_t zalign, unsigned offset) { - surf->level[level].npix_x = mip_minify(surf->npix_x, level); - surf->level[level].npix_y = mip_minify(surf->npix_y, level); - surf->level[level].npix_z = mip_minify(surf->npix_z, level); - surf->level[level].nblk_x = (surf->level[level].npix_x + surf->blk_w - 1) / surf->blk_w; - surf->level[level].nblk_y = (surf->level[level].npix_y + surf->blk_h - 1) / surf->blk_h; - surf->level[level].nblk_z = (surf->level[level].npix_z + surf->blk_d - 1) / surf->blk_d; - if (surf->nsamples == 1 && surf->level[level].mode == RADEON_SURF_MODE_2D) { - if (surf->level[level].nblk_x < xalign || surf->level[level].nblk_y < yalign) { - surf->level[level].mode = RADEON_SURF_MODE_1D; + surflevel->npix_x = mip_minify(surf->npix_x, level); + surflevel->npix_y = mip_minify(surf->npix_y, level); + surflevel->npix_z = mip_minify(surf->npix_z, level); + surflevel->nblk_x = (surflevel->npix_x + surf->blk_w - 1) / surf->blk_w; + surflevel->nblk_y = (surflevel->npix_y + surf->blk_h - 1) / surf->blk_h; + surflevel->nblk_z = (surflevel->npix_z + surf->blk_d - 1) / surf->blk_d; + if (surf->nsamples == 1 && surflevel->mode == RADEON_SURF_MODE_2D) { + if (surflevel->nblk_x < xalign || surflevel->nblk_y < yalign) { + surflevel->mode = RADEON_SURF_MODE_1D; return; } } - surf->level[level].nblk_x = ALIGN(surf->level[level].nblk_x, xalign); - surf->level[level].nblk_y = ALIGN(surf->level[level].nblk_y, yalign); - surf->level[level].nblk_z = ALIGN(surf->level[level].nblk_z, zalign); + surflevel->nblk_x = ALIGN(surflevel->nblk_x, xalign); + surflevel->nblk_y = ALIGN(surflevel->nblk_y, yalign); + surflevel->nblk_z = ALIGN(surflevel->nblk_z, zalign);

- surf->level[level].offset = offset; - surf->level[level].pitch_bytes = surf->level[level].nblk_x * surf->bpe * surf->nsamples; - surf->level[level].slice_size = surf->level[level].pitch_bytes * surf->level[level].nblk_y; + surflevel->offset = offset; + surflevel->pitch_bytes = surflevel->nblk_x * bpe * surf->nsamples; + surflevel->slice_size = surflevel->pitch_bytes * surflevel->nblk_y;

- surf->bo_size = offset + surf->level[level].slice_size * surf->level[level].nblk_z * surf->array_size; + surf->bo_size = offset + surflevel->slice_size * surflevel->nblk_z * surf->array_size; }

/* =========================================================================== @@ -264,7 +265,7 @@ static int r6_surface_init_linear(struct radeon_surface_manager *surf_man, /* build mipmap tree */ for (i = start_level; i <= surf->last_level; i++) { surf->level[i].mode = RADEON_SURF_MODE_LINEAR; - surf_minify(surf, i, xalign, yalign, zalign, offset); + surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset); /* level0 and first mipmap need to have alignment */ offset = surf->bo_size; if ((i == 0)) { @@ -292,7 +293,7 @@ static int r6_surface_init_linear_aligned(struct radeon_surface_manager *surf_ma /* build mipmap tree */ for (i = start_level; i <= surf->last_level; i++) { surf->level[i].mode = RADEON_SURF_MODE_LINEAR_ALIGNED; - surf_minify(surf, i, xalign, yalign, zalign, offset); + surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset); /* level0 and first mipmap need to have alignment */ offset = surf->bo_size; if ((i == 0)) { @@ -325,7 +326,7 @@ static int r6_surface_init_1d(struct radeon_surface_manager *surf_man, /* build mipmap tree */ for (i = start_level; i <= surf->last_level; i++) { surf->level[i].mode = RADEON_SURF_MODE_1D; - surf_minify(surf, i, xalign, yalign, zalign, offset); + surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset); /* level0 and first mipmap need to have alignment */ offset = surf->bo_size; if ((i == 0)) { @@ -363,7 +364,7 @@ static int r6_surface_init_2d(struct radeon_surface_manager *surf_man, /* build mipmap tree */ for (i = start_level; i <= surf->last_level; i++) { surf->level[i].mode = RADEON_SURF_MODE_2D; - surf_minify(surf, i, xalign, yalign, zalign, offset); + surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset); if (surf->level[i].mode == RADEON_SURF_MODE_1D) { return r6_surface_init_1d(surf_man, surf, offset, i); } @@ -543,6 +544,8 @@ static int eg_init_hw_info(struct radeon_surface_manager *surf_man) }

static void eg_surf_minify(struct radeon_surface *surf, + struct radeon_surface_level *surflevel, + unsigned bpe, unsigned level, unsigned slice_pt, unsigned mtilew, @@ -552,36 +555,38 @@ static void eg_surf_minify(struct radeon_surface *surf, { unsigned mtile_pr, mtile_ps;

- surf->level[level].npix_x = mip_minify(surf->npix_x, level); - surf->level[level].npix_y = mip_minify(surf->npix_y, level); - surf->level[level].npix_z = mip_minify(surf->npix_z, level); - surf->level[level].nblk_x = (surf->level[level].npix_x + surf->blk_w - 1) / surf->blk_w; - surf->level[level].nblk_y = (surf->level[level].npix_y + surf->blk_h - 1) / surf->blk_h; - surf->level[level].nblk_z = (surf->level[level].npix_z + surf->blk_d - 1) / surf->blk_d; - if (surf->nsamples == 1 && surf->level[level].mode == RADEON_SURF_MODE_2D) { - if (surf->level[level].nblk_x < mtilew || surf->level[level].nblk_y < mtileh) { - surf->level[level].mode = RADEON_SURF_MODE_1D; + surflevel->npix_x = mip_minify(surf->npix_x, level); + surflevel->npix_y = mip_minify(surf->npix_y, level); + surflevel->npix_z = mip_minify(surf->npix_z, level); + surflevel->nblk_x = (surflevel->npix_x + surf->blk_w - 1) / surf->blk_w; + surflevel->nblk_y = (surflevel->npix_y + surf->blk_h - 1) / surf->blk_h; + surflevel->nblk_z = (surflevel->npix_z + surf->blk_d - 1) / surf->blk_d; + if (surf->nsamples == 1 && surflevel->mode == RADEON_SURF_MODE_2D) { + if (surflevel->nblk_x < mtilew || surflevel->nblk_y < mtileh) { + surflevel->mode = RADEON_SURF_MODE_1D; return; } } - surf->level[level].nblk_x = ALIGN(surf->level[level].nblk_x, mtilew); - surf->level[level].nblk_y = ALIGN(surf->level[level].nblk_y, mtileh); - surf->level[level].nblk_z = ALIGN(surf->level[level].nblk_z, 1); + surflevel->nblk_x = ALIGN(surflevel->nblk_x, mtilew); + surflevel->nblk_y = ALIGN(surflevel->nblk_y, mtileh); + surflevel->nblk_z = ALIGN(surflevel->nblk_z, 1);

/* macro tile per row */ - mtile_pr = surf->level[level].nblk_x / mtilew; + mtile_pr = surflevel->nblk_x / mtilew; /* macro tile per slice */ - mtile_ps = (mtile_pr * surf->level[level].nblk_y) / mtileh; + mtile_ps = (mtile_pr * surflevel->nblk_y) / mtileh;

- surf->level[level].offset = offset; - surf->level[level].pitch_bytes = surf->level[level].nblk_x * surf->bpe * slice_pt; - surf->level[level].slice_size = mtile_ps * mtileb * slice_pt; + surflevel->offset = offset; + surflevel->pitch_bytes = surflevel->nblk_x * bpe * slice_pt; + surflevel->slice_size = mtile_ps * mtileb * slice_pt;

- surf->bo_size = offset + surf->level[level].slice_size * surf->level[level].nblk_z * surf->array_size; + surf->bo_size = offset + surflevel->slice_size * surflevel->nblk_z * surf->array_size; }

static int eg_surface_init_1d(struct radeon_surface_manager *surf_man, struct radeon_surface *surf, + struct radeon_surface_level *level, + unsigned bpe, uint64_t offset, unsigned start_level) { uint32_t xalign, yalign, zalign, tilew; @@ -589,45 +594,40 @@ static int eg_surface_init_1d(struct radeon_surface_manager *surf_man,

/* compute alignment */ tilew = 8; - xalign = surf_man->hw_info.group_bytes / (tilew * surf->bpe * surf->nsamples); - if (surf->flags & RADEON_SURF_SBUFFER) { - xalign = surf_man->hw_info.group_bytes / (tilew * surf->nsamples); - } + xalign = surf_man->hw_info.group_bytes / (tilew * bpe * surf->nsamples); xalign = MAX2(tilew, xalign); yalign = tilew; zalign = 1; if (surf->flags & RADEON_SURF_SCANOUT) { - xalign = MAX2((surf->bpe == 1) ? 64 : 32, xalign); + xalign = MAX2((bpe == 1) ? 64 : 32, xalign); } + if (!start_level) { - surf->bo_alignment = MAX2(256, surf_man->hw_info.group_bytes); + unsigned alignment = MAX2(256, surf_man->hw_info.group_bytes); + surf->bo_alignment = MAX2(surf->bo_alignment, alignment); + + if (offset) { + offset = ALIGN(offset, alignment); + } }

/* build mipmap tree */ for (i = start_level; i <= surf->last_level; i++) { - surf->level[i].mode = RADEON_SURF_MODE_1D; - surf_minify(surf, i, xalign, yalign, zalign, offset); + level[i].mode = RADEON_SURF_MODE_1D; + surf_minify(surf, level+i, bpe, i, xalign, yalign, zalign, offset); /* level0 and first mipmap need to have alignment */ offset = surf->bo_size; if ((i == 0)) { offset = ALIGN(offset, surf->bo_alignment); } } - - /* The depth and stencil buffers are in separate resources on evergreen. - * We allocate them in one buffer next to each other to simplify - * communication between the DDX and the Mesa driver. */ - if ((surf->flags & (RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) == - (RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) { - surf->stencil_offset = ALIGN(surf->bo_size, surf->bo_alignment); - surf->bo_size = surf->stencil_offset + surf->bo_size / 4; - } - return 0; }

static int eg_surface_init_2d(struct radeon_surface_manager *surf_man, struct radeon_surface *surf, + struct radeon_surface_level *level, + unsigned bpe, unsigned tile_split, uint64_t offset, unsigned start_level) { unsigned tilew, tileh, tileb; @@ -638,11 +638,11 @@ static int eg_surface_init_2d(struct radeon_surface_manager *surf_man, /* compute tile values */ tilew = 8; tileh = 8; - tileb = tilew * tileh * surf->bpe * surf->nsamples; + tileb = tilew * tileh * bpe * surf->nsamples; /* slices per tile */ slice_pt = 1; - if (tileb > surf->tile_split) { - slice_pt = tileb / surf->tile_split; + if (tileb > tile_split) { + slice_pt = tileb / tile_split; } tileb = tileb / slice_pt;

@@ -653,15 +653,20 @@ static int eg_surface_init_2d(struct radeon_surface_manager *surf_man, mtileb = (mtilew / tilew) * (mtileh / tileh) * tileb;

if (!start_level) { - surf->bo_alignment = MAX2(256, mtileb); + unsigned alignment = MAX2(256, mtileb); + surf->bo_alignment = MAX2(surf->bo_alignment, alignment); + + if (offset) { + offset = ALIGN(offset, alignment); + } }

/* build mipmap tree */ for (i = start_level; i <= surf->last_level; i++) { - surf->level[i].mode = RADEON_SURF_MODE_2D; - eg_surf_minify(surf, i, slice_pt, mtilew, mtileh, mtileb, offset); - if (surf->level[i].mode == RADEON_SURF_MODE_1D) { - return eg_surface_init_1d(surf_man, surf, offset, i); + level[i].mode = RADEON_SURF_MODE_2D; + eg_surf_minify(surf, level+i, bpe, i, slice_pt, mtilew, mtileh, mtileb, offset); + if (level[i].mode == RADEON_SURF_MODE_1D) { + return eg_surface_init_1d(surf_man, surf, level, bpe, offset, i); } /* level0 and first mipmap need to have alignment */ offset = surf->bo_size; @@ -669,13 +674,6 @@ static int eg_surface_init_2d(struct radeon_surface_manager *surf_man, offset = ALIGN(offset, surf->bo_alignment); } } - - if ((surf->flags & (RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) == - (RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) { - surf->stencil_offset = ALIGN(surf->bo_size, surf->bo_alignment); - surf->bo_size = surf->stencil_offset + surf->bo_size / 4; - } - return 0; }

@@ -762,6 +760,51 @@ static int eg_surface_sanity(struct radeon_surface_manager *surf_man, return 0; }

+static int eg_surface_init_1d_miptrees(struct radeon_surface_manager *surf_man, + struct radeon_surface *surf) +{ + unsigned zs_flags = RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER; + int r, is_depth_stencil = (surf->flags & zs_flags) == zs_flags; + /* Old libdrm headers didn't have stencil_level in it. This prevents crashes. */ + struct radeon_surface_level tmp[RADEON_SURF_MAX_LEVEL]; + struct radeon_surface_level *stencil_level = + (surf->flags & RADEON_SURF_HAS_SBUFFER_MIPTREE) ? surf->stencil_level : tmp; + + r = eg_surface_init_1d(surf_man, surf, surf->level, surf->bpe, 0, 0); + if (r) + return r; + + if (is_depth_stencil) { + r = eg_surface_init_1d(surf_man, surf, stencil_level, 1, + surf->bo_size, 0); + surf->stencil_offset = stencil_level[0].offset; + } + return r; +} + +static int eg_surface_init_2d_miptrees(struct radeon_surface_manager *surf_man, + struct radeon_surface *surf) +{ + unsigned zs_flags = RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER; + int r, is_depth_stencil = (surf->flags & zs_flags) == zs_flags; + /* Old libdrm headers didn't have stencil_level in it. This prevents crashes. */ + struct radeon_surface_level tmp[RADEON_SURF_MAX_LEVEL]; + struct radeon_surface_level *stencil_level = + (surf->flags & RADEON_SURF_HAS_SBUFFER_MIPTREE) ? surf->stencil_level : tmp; + + r = eg_surface_init_2d(surf_man, surf, surf->level, surf->bpe, + surf->tile_split, 0, 0); + if (r) + return r; + + if (is_depth_stencil) { + r = eg_surface_init_2d(surf_man, surf, stencil_level, 1, + surf->stencil_tile_split, surf->bo_size, 0); + surf->stencil_offset = stencil_level[0].offset; + } + return r; +} + static int eg_surface_init(struct radeon_surface_manager *surf_man, struct radeon_surface *surf) { @@ -797,6 +840,7 @@ static int eg_surface_init(struct radeon_surface_manager *surf_man, }

surf->stencil_offset = 0; + surf->bo_alignment = 0;

/* check tiling mode */ switch (mode) { @@ -807,10 +851,10 @@ static int eg_surface_init(struct radeon_surface_manager *surf_man, r = r6_surface_init_linear_aligned(surf_man, surf, 0, 0); break; case RADEON_SURF_MODE_1D: - r = eg_surface_init_1d(surf_man, surf, 0, 0); + r = eg_surface_init_1d_miptrees(surf_man, surf); break; case RADEON_SURF_MODE_2D: - r = eg_surface_init_2d(surf_man, surf, 0, 0); + r = eg_surface_init_2d_miptrees(surf_man, surf); break; default: return -EINVAL; @@ -1058,6 +1102,7 @@ static int si_surface_init(struct radeon_surface_manager *surf_man, }

surf->stencil_offset = 0; + surf->bo_alignment = 0;

/* check tiling mode */ switch (mode) { @@ -1068,10 +1113,10 @@ static int si_surface_init(struct radeon_surface_manager *surf_man, r = si_surface_init_linear_aligned(surf_man, surf, 0, 0); break; case RADEON_SURF_MODE_1D: - r = eg_surface_init_1d(surf_man, surf, 0, 0); + r = eg_surface_init_1d_miptrees(surf_man, surf); break; case RADEON_SURF_MODE_2D: - r = eg_surface_init_2d(surf_man, surf, 0, 0); + r = eg_surface_init_2d_miptrees(surf_man, surf); break; default: return -EINVAL; diff --git a/radeon/radeon_surface.h b/radeon/radeon_surface.h index bfee8ab..ed4b994 100644 --- a/radeon/radeon_surface.h +++ b/radeon/radeon_surface.h @@ -54,6 +54,7 @@ #define RADEON_SURF_SCANOUT (1 << 16) #define RADEON_SURF_ZBUFFER (1 << 17) #define RADEON_SURF_SBUFFER (1 << 18) +#define RADEON_SURF_HAS_SBUFFER_MIPTREE (1 << 19)

#define RADEON_SURF_GET(v, field) (((v) >> RADEON_SURF_ ## field ## _SHIFT) & RADEON_SURF_ ## field ## _MASK) #define RADEON_SURF_SET(v, field) (((v) & RADEON_SURF_ ## field ## _MASK) << RADEON_SURF_ ## field ## _SHIFT) @@ -102,6 +103,7 @@ struct radeon_surface { uint32_t stencil_tile_split; uint64_t stencil_offset; struct radeon_surface_level level[RADEON_SURF_MAX_LEVEL]; + struct radeon_surface_level stencil_level[RADEON_SURF_MAX_LEVEL]; };

struct radeon_surface_manager *radeon_surface_manager_new(int fd);