Flat-CCS eviction enhancements
v3: Incorporated the review suggestions [Matt]
Ramalingam C (3): drm/i915/gt: BUG_ON unexpected NULL at scatterlist walking drm/i915/gt: optimize the ccs_sz calculation per chunk drm/i915/gt: Document the eviction of the Flat-CCS objects
drivers/gpu/drm/i915/gt/intel_migrate.c | 73 ++++++++++++++----------- 1 file changed, 40 insertions(+), 33 deletions(-)
While locating the start of ccs scatterlist in smem scatterlist, that has to be the size of lmem obj size + corresponding ccs data size, report bug if scatterlist terminate before that length.
v2: s/GEM_BUG_ON/BUG_ON with more commenting [Matt] v3: Converted GEM_BUG_ON into BUG_ON with more documentation [Matt]
Signed-off-by: Ramalingam C ramalingam.c@intel.com Reviewed-by: Matthew Auld matthew.auld@intel.com (v1) --- drivers/gpu/drm/i915/gt/intel_migrate.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c index 9d552f30b627..168d17b6f48a 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.c +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -687,6 +687,16 @@ static void get_ccs_sg_sgt(struct sgt_dma *it, u32 bytes_to_cpy) bytes_to_cpy -= len;
it->sg = __sg_next(it->sg); + + /* + * On Flat-CCS capable platform when we back the lmem pages with + * smem pages we add extra pages at the end of the smem + * scatterlist, to store the ccs data corresponding to the lmem + * pages. get_ccs_sg_sgt() is called to get the pointer for the + * start of the extra pages added at the end of smem scatterlist. + * So scatterlist can't end at or before bytes_to_cpy. + */ + BUG_ON(!it->sg); it->dma = sg_dma_address(it->sg); it->max = it->dma + sg_dma_len(it->sg); } while (bytes_to_cpy); @@ -748,8 +758,10 @@ intel_context_migrate_copy(struct intel_context *ce, * Need to fix it. */ ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0; - if (ccs_bytes_to_cpy) + if (ccs_bytes_to_cpy) { + WARN_ON(abs(src_sz - dst_sz) < ccs_bytes_to_cpy); get_ccs_sg_sgt(&it_ccs, bytes_to_cpy); + } }
src_offset = 0;
On Mon, 02 May 2022, Ramalingam C ramalingam.c@intel.com wrote:
While locating the start of ccs scatterlist in smem scatterlist, that has to be the size of lmem obj size + corresponding ccs data size, report bug if scatterlist terminate before that length.
v2: s/GEM_BUG_ON/BUG_ON with more commenting [Matt] v3: Converted GEM_BUG_ON into BUG_ON with more documentation [Matt]
Signed-off-by: Ramalingam C ramalingam.c@intel.com Reviewed-by: Matthew Auld matthew.auld@intel.com (v1)
drivers/gpu/drm/i915/gt/intel_migrate.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c index 9d552f30b627..168d17b6f48a 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.c +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -687,6 +687,16 @@ static void get_ccs_sg_sgt(struct sgt_dma *it, u32 bytes_to_cpy) bytes_to_cpy -= len;
it->sg = __sg_next(it->sg);
/** On Flat-CCS capable platform when we back the lmem pages with* smem pages we add extra pages at the end of the smem* scatterlist, to store the ccs data corresponding to the lmem* pages. get_ccs_sg_sgt() is called to get the pointer for the* start of the extra pages added at the end of smem scatterlist.* So scatterlist can't end at or before bytes_to_cpy.*/BUG_ON(!it->sg);
Why would you have to bring the entire kernel down in this case? Why not just let it oops on the NULL pointer dereference?
I'd prefer nuking *all* of the current BUG/BUG_ON in the driver, and not add any single one back.
BR, Jani.
it->dma = sg_dma_address(it->sg); it->max = it->dma + sg_dma_len(it->sg);} while (bytes_to_cpy); @@ -748,8 +758,10 @@ intel_context_migrate_copy(struct intel_context *ce, * Need to fix it. */ ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0;
if (ccs_bytes_to_cpy)
if (ccs_bytes_to_cpy) {WARN_ON(abs(src_sz - dst_sz) < ccs_bytes_to_cpy); get_ccs_sg_sgt(&it_ccs, bytes_to_cpy);}}
src_offset = 0;
Calculate the ccs_sz that needs to be emitted based on the src and dst pages emitted per chunk. And handle the return value of emit_pte for the ccs pages.
v2: ccs_sz moved to the reduced scope [Matt]
Signed-off-by: Ramalingam C ramalingam.c@intel.com Reviewed-by: Matthew Auld matthew.auld@intel.com --- drivers/gpu/drm/i915/gt/intel_migrate.c | 36 +++++++++---------------- 1 file changed, 13 insertions(+), 23 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c index 168d17b6f48a..fc6975e55fae 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.c +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -647,17 +647,9 @@ static int scatter_list_length(struct scatterlist *sg)
static void calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem, - int *src_sz, int *ccs_sz, u32 bytes_to_cpy, - u32 ccs_bytes_to_cpy) + int *src_sz, u32 bytes_to_cpy, u32 ccs_bytes_to_cpy) { if (ccs_bytes_to_cpy) { - /* - * We can only copy the ccs data corresponding to - * the CHUNK_SZ of lmem which is - * GET_CCS_BYTES(i915, CHUNK_SZ)) - */ - *ccs_sz = min_t(int, ccs_bytes_to_cpy, GET_CCS_BYTES(i915, CHUNK_SZ)); - if (!src_is_lmem) /* * When CHUNK_SZ is passed all the pages upto CHUNK_SZ @@ -717,10 +709,10 @@ intel_context_migrate_copy(struct intel_context *ce, struct drm_i915_private *i915 = ce->engine->i915; u32 ccs_bytes_to_cpy = 0, bytes_to_cpy; enum i915_cache_level ccs_cache_level; - int src_sz, dst_sz, ccs_sz; u32 src_offset, dst_offset; u8 src_access, dst_access; struct i915_request *rq; + int src_sz, dst_sz; bool ccs_is_src; int err;
@@ -803,7 +795,7 @@ intel_context_migrate_copy(struct intel_context *ce, if (err) goto out_rq;
- calculate_chunk_sz(i915, src_is_lmem, &src_sz, &ccs_sz, + calculate_chunk_sz(i915, src_is_lmem, &src_sz, bytes_to_cpy, ccs_bytes_to_cpy);
len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, @@ -837,37 +829,35 @@ intel_context_migrate_copy(struct intel_context *ce, bytes_to_cpy -= len;
if (ccs_bytes_to_cpy) { + int ccs_sz; + err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); if (err) goto out_rq;
+ ccs_sz = GET_CCS_BYTES(i915, len); err = emit_pte(rq, &it_ccs, ccs_cache_level, false, ccs_is_src ? src_offset : dst_offset, ccs_sz); + if (err < 0) + goto out_rq; + if (err < ccs_sz) { + err = -EINVAL; + goto out_rq; + }
err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); if (err) goto out_rq;
- /* - * Using max of src_sz and dst_sz, as we need to - * pass the lmem size corresponding to the ccs - * blocks we need to handle. - */ - ccs_sz = max_t(int, ccs_is_src ? ccs_sz : src_sz, - ccs_is_src ? dst_sz : ccs_sz); - err = emit_copy_ccs(rq, dst_offset, dst_access, - src_offset, src_access, ccs_sz); + src_offset, src_access, len); if (err) goto out_rq;
err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); if (err) goto out_rq; - - /* Converting back to ccs bytes */ - ccs_sz = GET_CCS_BYTES(rq->engine->i915, ccs_sz); ccs_bytes_to_cpy -= ccs_sz; }
Capture the eviction details for Flat-CCS capable, lmem objects.
v2: Fix the Flat-ccs capbility of lmem obj with smem residency possibility [Thomas] v3: Fixed the suggestions [Matt]
Signed-off-by: Ramalingam C ramalingam.c@intel.com cc: Thomas Hellstrom thomas.hellstrom@linux.intel.com cc: Matthew Auld matthew.auld@intel.com Reviewed-by: Matthew Auld matthew.auld@intel.com --- drivers/gpu/drm/i915/gt/intel_migrate.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c index fc6975e55fae..509955885b93 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.c +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -485,16 +485,21 @@ static bool wa_1209644611_applies(int ver, u32 size) * And CCS data can be copied in and out of CCS region through * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly. * - * When we exhaust the lmem, if the object's placements support smem, then we can - * directly decompress the compressed lmem object into smem and start using it - * from smem itself. + * I915 supports Flat-CCS on lmem only objects. When an objects has smem in + * its preference list, on memory pressure, i915 needs to migrate the lmem + * content into smem. If the lmem object is Flat-CCS compressed by userspace, + * then i915 needs to decompress it. But I915 lack the required information + * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects. * - * But when we need to swapout the compressed lmem object into a smem region - * though objects' placement doesn't support smem, then we copy the lmem content - * as it is into smem region along with ccs data (using XY_CTRL_SURF_COPY_BLT). - * When the object is referred, lmem content will be swaped in along with - * restoration of the CCS data (using XY_CTRL_SURF_COPY_BLT) at corresponding - * location. + * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can + * be temporarily evicted to smem, along with the auxiliary CCS state, where + * it can be potentially swapped-out at a later point, if required. + * If userspace later touches the evicted pages, then we always move + * the backing memory back to lmem, which includes restoring the saved CCS state, + * and potentially performing any required swap-in. + * + * For the migration of the lmem objects with smem in placement list, such as + * {lmem, smem}, objects are treated as non Flat-CCS capable objects. */
static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
dri-devel@lists.freedesktop.org
-
Jani Nikula -
Ramalingam C