Change-Id: Ifb0dc95db6a358cf7f76e2a99f94c58637ad6ee6 Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 17 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 170 ++++++++++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 6 + drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 1 + 7 files changed, 187 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 4b66585a8638..f398a566f57b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -443,7 +443,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, int alignment, u32 initial_domain, u64 flags, enum ttm_bo_type type, struct reservation_object *resv, - struct drm_gem_object **obj); + struct drm_gem_object **obj, + struct ttm_process *process);
int amdgpu_mode_dumb_create(struct drm_file *file_priv, struct drm_device *dev, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c index ff89e84b34ce..1cee2125f570 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c @@ -146,7 +146,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev, AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED, - true, NULL, &gobj); + true, NULL, &gobj, &adev->kernel_process); if (ret) { pr_err("failed to allocate framebuffer (%d)\n", aligned_size); return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 37b19ce97699..d5cbdc810aba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -45,7 +45,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, int alignment, u32 initial_domain, u64 flags, enum ttm_bo_type type, struct reservation_object *resv, - struct drm_gem_object **obj) + struct drm_gem_object **obj, + struct ttm_process *process) { struct amdgpu_bo *bo; int r; @@ -56,8 +57,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, alignment = PAGE_SIZE; }
- r = amdgpu_bo_create(adev, size, alignment, initial_domain, - flags, type, resv, &bo); + r = amdgpu_bo_create1(adev, size, alignment, initial_domain, + flags, type, resv, &bo, process); if (r) { DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n", size, initial_domain, alignment, r); @@ -243,7 +244,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
r = amdgpu_gem_object_create(adev, size, args->in.alignment, (u32)(0xffffffff & args->in.domains), - flags, false, resv, &gobj); + flags, false, resv, &gobj, &vm->ttm_vm); if (!r) { abo = gem_to_amdgpu_bo(gobj); abo->tbo.process = &vm->ttm_vm; @@ -273,6 +274,8 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, { struct ttm_operation_ctx ctx = { true, false }; struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; struct drm_amdgpu_gem_userptr *args = data; struct drm_gem_object *gobj; struct amdgpu_bo *bo; @@ -297,7 +300,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
/* create a gem object to contain this object in */ r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU, - 0, 0, NULL, &gobj); + 0, 0, NULL, &gobj, &vm->ttm_vm); if (r) return r;
@@ -735,6 +738,8 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, struct drm_mode_create_dumb *args) { struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = file_priv->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; struct drm_gem_object *gobj; uint32_t handle; int r; @@ -747,7 +752,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_VRAM, AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, - false, NULL, &gobj); + false, NULL, &gobj, &vm->ttm_vm); if (r) return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e911db2d1945..7bb6ee777067 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -421,6 +421,11 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size, bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; #endif
+ if (type == ttm_bo_type_kernel) { + bo->tbo.priority = 1; + bo->tbo.process = &adev->kernel_process; + } + bo->tbo.bdev = &adev->mman.bdev; amdgpu_ttm_placement_from_domain(bo, domains); r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, @@ -447,11 +452,6 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size, else amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
- if (type == ttm_bo_type_kernel) { - bo->tbo.priority = 1; - bo->tbo.process = &adev->kernel_process; - } - if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { struct dma_fence *fence; @@ -538,6 +538,166 @@ int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size,
return r; } +static int amdgpu_bo_do_create1(struct amdgpu_device *adev, unsigned long size, + int byte_align, u32 domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct amdgpu_bo **bo_ptr, + struct ttm_process *process) +{ + struct ttm_operation_ctx ctx = { + .interruptible = (type != ttm_bo_type_kernel), + .no_wait_gpu = false, + .resv = resv, + .flags = TTM_OPT_FLAG_ALLOW_RES_EVICT + }; + struct amdgpu_bo *bo; + unsigned long page_align; + size_t acc_size; + u32 domains, preferred_domains, allowed_domains; + int r; + + page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; + size = ALIGN(size, PAGE_SIZE); + + if (!amdgpu_bo_validate_size(adev, size, domain)) + return -ENOMEM; + + *bo_ptr = NULL; + + acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, + sizeof(struct amdgpu_bo)); + + preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM | + AMDGPU_GEM_DOMAIN_GTT | + AMDGPU_GEM_DOMAIN_CPU | + AMDGPU_GEM_DOMAIN_GDS | + AMDGPU_GEM_DOMAIN_GWS | + AMDGPU_GEM_DOMAIN_OA); + allowed_domains = preferred_domains; + if (type != ttm_bo_type_kernel && + allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) + allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; + domains = preferred_domains; +retry: + bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); + if (bo == NULL) + return -ENOMEM; + drm_gem_private_object_init(adev->ddev, &bo->gem_base, size); + INIT_LIST_HEAD(&bo->shadow_list); + INIT_LIST_HEAD(&bo->va); + bo->preferred_domains = preferred_domains; + bo->allowed_domains = allowed_domains; + + bo->flags = flags; + +#ifdef CONFIG_X86_32 + /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit + * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 + */ + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) + /* Don't try to enable write-combining when it can't work, or things + * may be slow + * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 + */ + +#ifndef CONFIG_COMPILE_TEST +#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ + thanks to write-combining +#endif + + if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) + DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " + "better performance thanks to write-combining\n"); + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#else + /* For architectures that don't support WC memory, + * mask out the WC flag from the BO + */ + if (!drm_arch_can_wc_memory()) + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#endif + + if (type == ttm_bo_type_kernel) + bo->tbo.priority = 1; + bo->tbo.process = process; + + bo->tbo.bdev = &adev->mman.bdev; + amdgpu_ttm_placement_from_domain(bo, domains); + r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, + &bo->placement, page_align, &ctx, acc_size, + NULL, resv, &amdgpu_ttm_bo_destroy); + if (unlikely(r && r != -ERESTARTSYS) && type == ttm_bo_type_device && + !(flags & AMDGPU_GEM_CREATE_NO_FALLBACK)) { + if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { + flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + goto retry; + } else if (domains != allowed_domains) { + domains = allowed_domains; + goto retry; + } + } + if (unlikely(r)) + return r; + + if (adev->gmc.visible_vram_size < adev->gmc.real_vram_size && + bo->tbo.mem.mem_type == TTM_PL_VRAM && + bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT) + amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, + ctx.bytes_moved); + else + amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); + + if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && + bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { + struct dma_fence *fence; + + r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence); + if (unlikely(r)) + goto fail_unreserve; + + amdgpu_bo_fence(bo, fence, false); + dma_fence_put(bo->tbo.moving); + bo->tbo.moving = dma_fence_get(fence); + dma_fence_put(fence); + } + if (!resv) + amdgpu_bo_unreserve(bo); + *bo_ptr = bo; + + trace_amdgpu_bo_create(bo); + + /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ + if (type == ttm_bo_type_device) + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + + return 0; + +fail_unreserve: + if (!resv) + ww_mutex_unlock(&bo->tbo.resv->lock); + amdgpu_bo_unref(&bo); + return r; +} + + +int amdgpu_bo_create1(struct amdgpu_device *adev, unsigned long size, + int byte_align, u32 domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct amdgpu_bo **bo_ptr, struct ttm_process *process) +{ + uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW; + int r; + + r = amdgpu_bo_do_create1(adev, size, byte_align, domain, + parent_flags, type, resv, bo_ptr, process); + if (r) + return r; + return r; +} +
int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, struct amdgpu_ring *ring, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 546f77cb7882..d4191f0424d1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -208,6 +208,12 @@ int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size, u64 flags, enum ttm_bo_type type, struct reservation_object *resv, struct amdgpu_bo **bo_ptr); +int amdgpu_bo_create1(struct amdgpu_device *adev, unsigned long size, + int byte_align, u32 domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct amdgpu_bo **bo_ptr, struct ttm_process *process); + int amdgpu_bo_create_reserved(struct amdgpu_device *adev, unsigned long size, int align, u32 domain, struct amdgpu_bo **bo_ptr, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 4c9e10505e2d..591db2df1685 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1535,6 +1535,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) return;
amdgpu_ttm_debugfs_fini(adev); + ttm_process_fini(&adev->kernel_process, &adev->mman.bdev); amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); amdgpu_ttm_fw_reserve_vram_fini(adev); if (adev->mman.aper_base_kaddr) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 6ef449ea8d07..3a54b236fd42 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2612,6 +2612,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_vm_free_mapping(adev, vm, mapping, NULL); }
+ ttm_process_fini(&vm->ttm_vm, &adev->mman.bdev); root = amdgpu_bo_ref(vm->root.base.bo); r = amdgpu_bo_reserve(root, true); if (r) {