amdkfd should set interrupts for pipes 1-7.

Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/cik.c | 71 +------------------------------------------- 1 file changed, 1 insertion(+), 70 deletions(-)

diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index 9571be8..d54d3d7 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -7265,8 +7265,7 @@ static int cik_irq_init(struct radeon_device *rdev) int cik_irq_set(struct radeon_device *rdev) { u32 cp_int_cntl; - u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3; - u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3; + u32 cp_m1p0; u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; u32 grbm_int_cntl = 0; @@ -7300,13 +7299,6 @@ int cik_irq_set(struct radeon_device *rdev) dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;

cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; - cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; - cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; - cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; - cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; - cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; - cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; - cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;

if (rdev->flags & RADEON_IS_IGP) thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) & @@ -7328,33 +7320,6 @@ int cik_irq_set(struct radeon_device *rdev) case 0: cp_m1p0 |= TIME_STAMP_INT_ENABLE; break; - case 1: - cp_m1p1 |= TIME_STAMP_INT_ENABLE; - break; - case 2: - cp_m1p2 |= TIME_STAMP_INT_ENABLE; - break; - case 3: - cp_m1p2 |= TIME_STAMP_INT_ENABLE; - break; - default: - DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); - break; - } - } else if (ring->me == 2) { - switch (ring->pipe) { - case 0: - cp_m2p0 |= TIME_STAMP_INT_ENABLE; - break; - case 1: - cp_m2p1 |= TIME_STAMP_INT_ENABLE; - break; - case 2: - cp_m2p2 |= TIME_STAMP_INT_ENABLE; - break; - case 3: - cp_m2p2 |= TIME_STAMP_INT_ENABLE; - break; default: DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); break; @@ -7371,33 +7336,6 @@ int cik_irq_set(struct radeon_device *rdev) case 0: cp_m1p0 |= TIME_STAMP_INT_ENABLE; break; - case 1: - cp_m1p1 |= TIME_STAMP_INT_ENABLE; - break; - case 2: - cp_m1p2 |= TIME_STAMP_INT_ENABLE; - break; - case 3: - cp_m1p2 |= TIME_STAMP_INT_ENABLE; - break; - default: - DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); - break; - } - } else if (ring->me == 2) { - switch (ring->pipe) { - case 0: - cp_m2p0 |= TIME_STAMP_INT_ENABLE; - break; - case 1: - cp_m2p1 |= TIME_STAMP_INT_ENABLE; - break; - case 2: - cp_m2p2 |= TIME_STAMP_INT_ENABLE; - break; - case 3: - cp_m2p2 |= TIME_STAMP_INT_ENABLE; - break; default: DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); break; @@ -7486,13 +7424,6 @@ int cik_irq_set(struct radeon_device *rdev) WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);

WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); - WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1); - WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2); - WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3); - WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0); - WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1); - WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2); - WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);

WREG32(GRBM_INT_CNTL, grbm_int_cntl);

Implementing a lock for selecting and accessing shader engines and arrays. This lock will make sure that radeon and amdkfd are not colliding when accessing shader engines and arrays with GRBM_GFX_INDEX register.

Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/cik.c | 26 ++++++++++++++++++++++++++ drivers/gpu/drm/radeon/radeon.h | 2 ++ drivers/gpu/drm/radeon/radeon_device.c | 1 + 3 files changed, 29 insertions(+)

diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index d54d3d7..0096538 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -1563,6 +1563,8 @@ static const u32 godavari_golden_registers[] =

static void cik_init_golden_registers(struct radeon_device *rdev) { + /* Some of the registers might be dependant on GRBM_GFX_INDEX */ + mutex_lock(&rdev->grbm_idx_mutex); switch (rdev->family) { case CHIP_BONAIRE: radeon_program_register_sequence(rdev, @@ -1637,6 +1639,7 @@ static void cik_init_golden_registers(struct radeon_device *rdev) default: break; } + mutex_unlock(&rdev->grbm_idx_mutex); }

/** @@ -3419,6 +3422,7 @@ static void cik_setup_rb(struct radeon_device *rdev, u32 disabled_rbs = 0; u32 enabled_rbs = 0;

+ mutex_lock(&rdev->grbm_idx_mutex); for (i = 0; i < se_num; i++) { for (j = 0; j < sh_per_se; j++) { cik_select_se_sh(rdev, i, j); @@ -3430,6 +3434,7 @@ static void cik_setup_rb(struct radeon_device *rdev, } } cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex);

mask = 1; for (i = 0; i < max_rb_num_per_se * se_num; i++) { @@ -3440,6 +3445,7 @@ static void cik_setup_rb(struct radeon_device *rdev,

rdev->config.cik.backend_enable_mask = enabled_rbs;

+ mutex_lock(&rdev->grbm_idx_mutex); for (i = 0; i < se_num; i++) { cik_select_se_sh(rdev, i, 0xffffffff); data = 0; @@ -3467,6 +3473,7 @@ static void cik_setup_rb(struct radeon_device *rdev, WREG32(PA_SC_RASTER_CONFIG, data); } cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex); }

/** @@ -3684,6 +3691,12 @@ static void cik_gpu_init(struct radeon_device *rdev) /* set HW defaults for 3D engine */ WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));

+ mutex_lock(&rdev->grbm_idx_mutex); + /* + * making sure that the following register writes will be broadcasted + * to all the shaders + */ + cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); WREG32(SX_DEBUG_1, 0x20);

WREG32(TA_CNTL_AUX, 0x00010000); @@ -3739,6 +3752,7 @@ static void cik_gpu_init(struct radeon_device *rdev)

WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); + mutex_unlock(&rdev->grbm_idx_mutex);

udelay(50); } @@ -6036,6 +6050,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) u32 i, j, k; u32 mask;

+ mutex_lock(&rdev->grbm_idx_mutex); for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { cik_select_se_sh(rdev, i, j); @@ -6047,6 +6062,7 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) } } cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex);

mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; for (k = 0; k < rdev->usec_timeout; k++) { @@ -6181,10 +6197,12 @@ static int cik_rlc_resume(struct radeon_device *rdev) WREG32(RLC_LB_CNTR_INIT, 0); WREG32(RLC_LB_CNTR_MAX, 0x00008000);

+ mutex_lock(&rdev->grbm_idx_mutex); cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); WREG32(RLC_LB_PARAMS, 0x00600408); WREG32(RLC_LB_CNTL, 0x80000004); + mutex_unlock(&rdev->grbm_idx_mutex);

WREG32(RLC_MC_CNTL, 0); WREG32(RLC_UCODE_CNTL, 0); @@ -6251,11 +6269,13 @@ static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)

tmp = cik_halt_rlc(rdev);

+ mutex_lock(&rdev->grbm_idx_mutex); cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; WREG32(RLC_SERDES_WR_CTRL, tmp2); + mutex_unlock(&rdev->grbm_idx_mutex);

cik_update_rlc(rdev, tmp);

@@ -6297,11 +6317,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)

tmp = cik_halt_rlc(rdev);

+ mutex_lock(&rdev->grbm_idx_mutex); cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; WREG32(RLC_SERDES_WR_CTRL, data); + mutex_unlock(&rdev->grbm_idx_mutex);

cik_update_rlc(rdev, tmp);

@@ -6345,11 +6367,13 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)

tmp = cik_halt_rlc(rdev);

+ mutex_lock(&rdev->grbm_idx_mutex); cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; WREG32(RLC_SERDES_WR_CTRL, data); + mutex_unlock(&rdev->grbm_idx_mutex);

cik_update_rlc(rdev, tmp); } @@ -6778,10 +6802,12 @@ static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) u32 mask = 0, tmp, tmp1; int i;

+ mutex_lock(&rdev->grbm_idx_mutex); cik_select_se_sh(rdev, se, sh); tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); + mutex_unlock(&rdev->grbm_idx_mutex);

tmp &= 0xffff0000;

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 75bcc04..e420557 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -2374,6 +2374,8 @@ struct radeon_device { struct radeon_atcs atcs; /* srbm instance registers */ struct mutex srbm_mutex; + /* GRBM index mutex. Protects concurrents access to GRBM index */ + struct mutex grbm_idx_mutex; /* clock, powergating flags */ u32 cg_flags; u32 pg_flags; diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 827bcd1..b80100d 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -1298,6 +1298,7 @@ int radeon_device_init(struct radeon_device *rdev, mutex_init(&rdev->pm.mutex); mutex_init(&rdev->gpu_clock_mutex); mutex_init(&rdev->srbm_mutex); + mutex_init(&rdev->grbm_idx_mutex); init_rwsem(&rdev->pm.mclk_lock); init_rwsem(&rdev->exclusive_lock); init_waitqueue_head(&rdev->irq.vblank_queue);

Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- CREDITS | 7 +++++++ MAINTAINERS | 10 ++++++++++ 2 files changed, 17 insertions(+)

diff --git a/CREDITS b/CREDITS index 28ee151..e9628d5 100644 --- a/CREDITS +++ b/CREDITS @@ -1197,6 +1197,13 @@ S: R. Tocantins, 89 - Cristo Rei S: 80050-430 - Curitiba - Paraná S: Brazil

+N: Oded Gabbay +E: oded.gabbay@gmail.com +D: AMD KFD maintainer +S: 12 Shraga Raphaeli +S: Petah-Tikva, 4906418 +S: Israel + N: Kumar Gala E: galak@kernel.crashing.org D: Embedded PowerPC 6xx/7xx/74xx/82xx/83xx/85xx support diff --git a/MAINTAINERS b/MAINTAINERS index d76e077..da3aecb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -589,6 +589,16 @@ F: drivers/crypto/geode* F: drivers/video/geode/ F: arch/x86/include/asm/geode.h

+AMD KFD (radeon extension) +M: Oded Gabbay oded.gabbay@amd.com +L: dri-devel@lists.freedesktop.org +T: git git://people.freedesktop.org/~gabbayo/linux.git +S: Supported +F: drivers/gpu/drm/radeon/amdkfd/* +F: drivers/gpu/drm/radeon/radeon_kfd.c +F: drivers/gpu/drm/radeon/radeon_kfd.h +F: include/linux/uapi/linux/kfd_ioctl.h + AMD IOMMU (AMD-VI) M: Joerg Roedel joro@8bytes.org L: iommu@lists.linux-foundation.org

This patch adds the amdkfd skeleton driver. The driver does nothing except define a /dev/kfd device.

It returns -ENODEV on all amdkfd IOCTLs.

(v3) move bool to end of struct

(v3) remove pmc ioctls

(v3) add meaningful error message for ioctl error

Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/Kconfig | 2 + drivers/gpu/drm/radeon/Makefile | 2 + drivers/gpu/drm/radeon/amdkfd/Kconfig | 10 ++ drivers/gpu/drm/radeon/amdkfd/Makefile | 9 ++ drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c | 187 ++++++++++++++++++++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_device.c | 129 +++++++++++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_module.c | 98 +++++++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_priv.h | 82 ++++++++++++ 8 files changed, 519 insertions(+) create mode 100644 drivers/gpu/drm/radeon/amdkfd/Kconfig create mode 100644 drivers/gpu/drm/radeon/amdkfd/Makefile create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_device.c create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_module.c create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_priv.h

diff --git a/drivers/gpu/drm/radeon/Kconfig b/drivers/gpu/drm/radeon/Kconfig index 970f8e9..b697321 100644 --- a/drivers/gpu/drm/radeon/Kconfig +++ b/drivers/gpu/drm/radeon/Kconfig @@ -6,3 +6,5 @@ config DRM_RADEON_UMS

Userspace modesetting is deprecated for quite some time now, so enable this only if you have ancient versions of the DDX drivers. + +source "drivers/gpu/drm/radeon/amdkfd/Kconfig" diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile index 1476103..8ab6b6e 100644 --- a/drivers/gpu/drm/radeon/Makefile +++ b/drivers/gpu/drm/radeon/Makefile @@ -112,4 +112,6 @@ radeon-$(CONFIG_ACPI) += radeon_acpi.o

obj-$(CONFIG_DRM_RADEON)+= radeon.o

+obj-$(CONFIG_HSA_RADEON)+= amdkfd/ + CFLAGS_radeon_trace_points.o := -I$(src) diff --git a/drivers/gpu/drm/radeon/amdkfd/Kconfig b/drivers/gpu/drm/radeon/amdkfd/Kconfig new file mode 100644 index 0000000..900bb34 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/Kconfig @@ -0,0 +1,10 @@ +# +# Heterogenous system architecture configuration +# + +config HSA_RADEON + tristate "HSA kernel driver for AMD Radeon devices" + depends on DRM_RADEON && AMD_IOMMU_V2 && X86_64 + default m + help + Enable this if you want to use HSA features on AMD radeon devices. diff --git a/drivers/gpu/drm/radeon/amdkfd/Makefile b/drivers/gpu/drm/radeon/amdkfd/Makefile new file mode 100644 index 0000000..9564e75 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/Makefile @@ -0,0 +1,9 @@ +# +# Makefile for Heterogenous System Architecture support for AMD radeon devices +# + +ccflags-y := -Iinclude/drm + +amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o + +obj-$(CONFIG_HSA_RADEON) += amdkfd.o diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c new file mode 100644 index 0000000..f198e5a --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c @@ -0,0 +1,187 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/device.h> +#include <linux/export.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/compat.h> +#include <uapi/linux/kfd_ioctl.h> +#include <linux/time.h> +#include <linux/mm.h> +#include <linux/uaccess.h> +#include <uapi/asm-generic/mman-common.h> +#include <asm/processor.h> +#include "kfd_priv.h" + +static long kfd_ioctl(struct file *, unsigned int, unsigned long); +static int kfd_open(struct inode *, struct file *); + +static const char kfd_dev_name[] = "kfd"; + +static const struct file_operations kfd_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = kfd_ioctl, + .compat_ioctl = kfd_ioctl, + .open = kfd_open, +}; + +static int kfd_char_dev_major = -1; +static struct class *kfd_class; +struct device *kfd_device; + +int kfd_chardev_init(void) +{ + int err = 0; + + kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); + err = kfd_char_dev_major; + if (err < 0) + goto err_register_chrdev; + + kfd_class = class_create(THIS_MODULE, kfd_dev_name); + err = PTR_ERR(kfd_class); + if (IS_ERR(kfd_class)) + goto err_class_create; + + kfd_device = device_create(kfd_class, NULL, MKDEV(kfd_char_dev_major, 0), NULL, kfd_dev_name); + err = PTR_ERR(kfd_device); + if (IS_ERR(kfd_device)) + goto err_device_create; + + return 0; + +err_device_create: + class_destroy(kfd_class); +err_class_create: + unregister_chrdev(kfd_char_dev_major, kfd_dev_name); +err_register_chrdev: + return err; +} + +void kfd_chardev_exit(void) +{ + device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); + class_destroy(kfd_class); + unregister_chrdev(kfd_char_dev_major, kfd_dev_name); +} + +struct device *kfd_chardev(void) +{ + return kfd_device; +} + + +static int kfd_open(struct inode *inode, struct file *filep) +{ + if (iminor(inode) != 0) + return -ENODEV; + + return 0; +} + +static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, void __user *arg) +{ + return -ENODEV; +} + +static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, void __user *arg) +{ + return -ENODEV; +} + +static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, void __user *arg) +{ + return -ENODEV; +} + +static long kfd_ioctl_set_memory_policy(struct file *filep, struct kfd_process *p, void __user *arg) +{ + return -ENODEV; +} + +static long kfd_ioctl_get_clock_counters(struct file *filep, struct kfd_process *p, void __user *arg) +{ + return -ENODEV; +} + + +static int kfd_ioctl_get_process_apertures(struct file *filp, struct kfd_process *p, void __user *arg) +{ + return -ENODEV; +} + +static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) +{ + struct kfd_process *process; + long err = -EINVAL; + + dev_dbg(kfd_device, + "ioctl cmd 0x%x (#%d), arg 0x%lx\n", + cmd, _IOC_NR(cmd), arg); + + /* TODO: add function that retrieves process */ + process = NULL; + + switch (cmd) { + case KFD_IOC_CREATE_QUEUE: + err = kfd_ioctl_create_queue(filep, process, (void __user *)arg); + break; + + case KFD_IOC_DESTROY_QUEUE: + err = kfd_ioctl_destroy_queue(filep, process, (void __user *)arg); + break; + + case KFD_IOC_SET_MEMORY_POLICY: + err = kfd_ioctl_set_memory_policy(filep, process, (void __user *)arg); + break; + + case KFD_IOC_GET_CLOCK_COUNTERS: + err = kfd_ioctl_get_clock_counters(filep, process, (void __user *)arg); + break; + + case KFD_IOC_GET_PROCESS_APERTURES: + err = kfd_ioctl_get_process_apertures(filep, process, (void __user *)arg); + break; + + case KFD_IOC_UPDATE_QUEUE: + err = kfd_ioctl_update_queue(filep, process, (void __user *)arg); + break; + + default: + dev_err(kfd_device, + "unknown ioctl cmd 0x%x, arg 0x%lx)\n", + cmd, arg); + err = -EINVAL; + break; + } + + if (err < 0) + dev_err(kfd_device, + "ioctl error %ld for ioctl cmd 0x%x (#%d)\n", + err, cmd, _IOC_NR(cmd)); + + return err; +} diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_device.c b/drivers/gpu/drm/radeon/amdkfd/kfd_device.c new file mode 100644 index 0000000..dd63ce09 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_device.c @@ -0,0 +1,129 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/amd-iommu.h> +#include <linux/bsearch.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include "kfd_priv.h" + +static const struct kfd_device_info kaveri_device_info = { + .max_pasid_bits = 16, +}; + +struct kfd_deviceid { + unsigned short did; + const struct kfd_device_info *device_info; +}; + +/* Please keep this sorted by increasing device id. */ +static const struct kfd_deviceid supported_devices[] = { + { 0x1304, &kaveri_device_info }, /* Kaveri */ + { 0x1305, &kaveri_device_info }, /* Kaveri */ + { 0x1306, &kaveri_device_info }, /* Kaveri */ + { 0x1307, &kaveri_device_info }, /* Kaveri */ + { 0x1309, &kaveri_device_info }, /* Kaveri */ + { 0x130A, &kaveri_device_info }, /* Kaveri */ + { 0x130B, &kaveri_device_info }, /* Kaveri */ + { 0x130C, &kaveri_device_info }, /* Kaveri */ + { 0x130D, &kaveri_device_info }, /* Kaveri */ + { 0x130E, &kaveri_device_info }, /* Kaveri */ + { 0x130F, &kaveri_device_info }, /* Kaveri */ + { 0x1310, &kaveri_device_info }, /* Kaveri */ + { 0x1311, &kaveri_device_info }, /* Kaveri */ + { 0x1312, &kaveri_device_info }, /* Kaveri */ + { 0x1313, &kaveri_device_info }, /* Kaveri */ + { 0x1315, &kaveri_device_info }, /* Kaveri */ + { 0x1316, &kaveri_device_info }, /* Kaveri */ + { 0x1317, &kaveri_device_info }, /* Kaveri */ + { 0x1318, &kaveri_device_info }, /* Kaveri */ + { 0x131B, &kaveri_device_info }, /* Kaveri */ + { 0x131C, &kaveri_device_info }, /* Kaveri */ + { 0x131D, &kaveri_device_info }, /* Kaveri */ +}; + +static const struct kfd_device_info *lookup_device_info(unsigned short did) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(supported_devices); i++) { + if (supported_devices[i].did == did) { + BUG_ON(supported_devices[i].device_info == NULL); + return supported_devices[i].device_info; + } + } + + return NULL; +} + +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev) +{ + struct kfd_dev *kfd; + + const struct kfd_device_info *device_info = lookup_device_info(pdev->device); + + if (!device_info) + return NULL; + + kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); + if (!kfd) + return NULL; + + kfd->kgd = kgd; + kfd->device_info = device_info; + kfd->pdev = pdev; + + return kfd; +} + +bool kgd2kfd_device_init(struct kfd_dev *kfd, + const struct kgd2kfd_shared_resources *gpu_resources) +{ + kfd->shared_resources = *gpu_resources; + + kfd->init_complete = true; + dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor, + kfd->pdev->device); + + return true; +} + +void kgd2kfd_device_exit(struct kfd_dev *kfd) +{ + kfree(kfd); +} + +void kgd2kfd_suspend(struct kfd_dev *kfd) +{ + BUG_ON(kfd == NULL); +} + +int kgd2kfd_resume(struct kfd_dev *kfd) +{ + BUG_ON(kfd == NULL); + + return 0; +} + +void kgd2kfd_interrupt(struct kfd_dev *dev, const void *ih_ring_entry) +{ +} diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_module.c b/drivers/gpu/drm/radeon/amdkfd/kfd_module.c new file mode 100644 index 0000000..1223e9b --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_module.c @@ -0,0 +1,98 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/notifier.h> +#include <linux/moduleparam.h> +#include <linux/device.h> +#include "kfd_priv.h" + +#define KFD_DRIVER_AUTHOR "AMD Inc. and others" + +#define KFD_DRIVER_DESC "Standalone HSA driver for AMD's GPUs" +#define KFD_DRIVER_DATE "20140804" +#define KFD_DRIVER_MAJOR 0 +#define KFD_DRIVER_MINOR 6 +#define KFD_DRIVER_PATCHLEVEL 4 + +const struct kfd2kgd_calls *kfd2kgd; +static const struct kgd2kfd_calls kgd2kfd = { + .exit = kgd2kfd_exit, + .probe = kgd2kfd_probe, + .device_init = kgd2kfd_device_init, + .device_exit = kgd2kfd_device_exit, + .interrupt = kgd2kfd_interrupt, + .suspend = kgd2kfd_suspend, + .resume = kgd2kfd_resume, +}; + +bool kgd2kfd_init(unsigned interface_version, + const struct kfd2kgd_calls *f2g, + const struct kgd2kfd_calls **g2f) +{ + /* Only one interface version is supported, no kfd/kgd version skew allowed. */ + if (interface_version != KFD_INTERFACE_VERSION) + return false; + + kfd2kgd = f2g; + *g2f = &kgd2kfd; + + return true; +} +EXPORT_SYMBOL(kgd2kfd_init); + +void kgd2kfd_exit(void) +{ +} + +static int __init kfd_module_init(void) +{ + int err; + + err = kfd_chardev_init(); + if (err < 0) + goto err_ioctl; + + dev_info(kfd_device, "Initialized module\n"); + + return 0; + +err_ioctl: + return err; +} + +static void __exit kfd_module_exit(void) +{ + kfd_chardev_exit(); + dev_info(kfd_device, "Removed module\n"); +} + +module_init(kfd_module_init); +module_exit(kfd_module_exit); + +MODULE_AUTHOR(KFD_DRIVER_AUTHOR); +MODULE_DESCRIPTION(KFD_DRIVER_DESC); +MODULE_LICENSE("GPL and additional rights"); +MODULE_VERSION(__stringify(KFD_DRIVER_MAJOR) "." + __stringify(KFD_DRIVER_MINOR) "." + __stringify(KFD_DRIVER_PATCHLEVEL)); diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h new file mode 100644 index 0000000..790a221 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h @@ -0,0 +1,82 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef KFD_PRIV_H_INCLUDED +#define KFD_PRIV_H_INCLUDED + +#include <linux/hashtable.h> +#include <linux/mmu_notifier.h> +#include <linux/mutex.h> +#include <linux/types.h> +#include <linux/atomic.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> +#include "../radeon_kfd.h" + +struct kfd_device_info { + const struct kfd_scheduler_class *scheduler_class; + unsigned int max_pasid_bits; + size_t ih_ring_entry_size; +}; + +struct kfd_dev { + struct kgd_dev *kgd; + + const struct kfd_device_info *device_info; + struct pci_dev *pdev; + + unsigned int id; /* topology stub index */ + + struct kgd2kfd_shared_resources shared_resources; + + bool init_complete; + +}; + +/* KGD2KFD callbacks */ +void kgd2kfd_exit(void); +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev); +bool kgd2kfd_device_init(struct kfd_dev *kfd, + const struct kgd2kfd_shared_resources *gpu_resources); +void kgd2kfd_device_exit(struct kfd_dev *kfd); + +extern const struct kfd2kgd_calls *kfd2kgd; + +/* Character device interface */ +int kfd_chardev_init(void); +void kfd_chardev_exit(void); +struct device *kfd_chardev(void); + +/* Process data */ +struct kfd_process { +}; + +extern struct device *kfd_device; + +/* Interrupts */ +void kgd2kfd_interrupt(struct kfd_dev *dev, const void *ih_ring_entry); + +/* Power Management */ +void kgd2kfd_suspend(struct kfd_dev *dev); +int kgd2kfd_resume(struct kfd_dev *dev); + +#endif

From: Andrew Lewycky Andrew.Lewycky@amd.com

This patch adds the process module and three helper modules:

- kfd_process, which handles process which open /dev/kfd

- kfd_doorbell, which provides helper functions for doorbell allocation, release and mapping to userspace

- kfd_pasid, which provides helper functions for pasid allocation and release

- kfd_aperture, which provides helper functions for managing the LDS, Local GPU memory and Scratch memory apertures of the process

This patch only contains the basic kfd_process module, which doesn't contain the reference to the queue scheduler. This was done to allow easier code review.

Also, this patch doesn't contain the calls to the IOMMU driver for binding the pasid to the device. Again, this was done to allow easier code review

The kfd_process object is created when a process opens /dev/kfd and is closed when the mm_struct of that process is teared-down.

v3: Remove kfd_vidmem file v3: Replace direct mmput call to mmu_notifier release v3: remove typedefs v3: move bool to end of struct v3: Add new kernel params for gart usage limitation v3: init sa manager v3: fix debug msgs v3: remove support for LDS in 32 bit v3: Change code to support mmap of doorbell pages from userspace v3: Add documentation for apertures

Signed-off-by: Andrew Lewycky Andrew.Lewycky@amd.com Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/amdkfd/Makefile | 4 +- drivers/gpu/drm/radeon/amdkfd/kfd_aperture.c | 350 +++++++++++++++++++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c | 31 ++- drivers/gpu/drm/radeon/amdkfd/kfd_device.c | 45 +++- drivers/gpu/drm/radeon/amdkfd/kfd_doorbell.c | 236 ++++++++++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_module.c | 32 ++- drivers/gpu/drm/radeon/amdkfd/kfd_pasid.c | 95 ++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_priv.h | 141 +++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_process.c | 319 ++++++++++++++++++++++++ drivers/gpu/drm/radeon/radeon_kfd.c | 21 +- 10 files changed, 1247 insertions(+), 27 deletions(-) create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_aperture.c create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_doorbell.c create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_pasid.c create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_process.c

diff --git a/drivers/gpu/drm/radeon/amdkfd/Makefile b/drivers/gpu/drm/radeon/amdkfd/Makefile index 08ecfcd..6d6746e 100644 --- a/drivers/gpu/drm/radeon/amdkfd/Makefile +++ b/drivers/gpu/drm/radeon/amdkfd/Makefile @@ -4,6 +4,8 @@

ccflags-y := -Iinclude/drm

-amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o +amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ + kfd_pasid.o kfd_doorbell.o kfd_aperture.o \ + kfd_process.o

obj-$(CONFIG_HSA_RADEON) += amdkfd.o diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_aperture.c b/drivers/gpu/drm/radeon/amdkfd/kfd_aperture.c new file mode 100644 index 0000000..8cfb720 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_aperture.c @@ -0,0 +1,350 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/device.h> +#include <linux/export.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/compat.h> +#include <uapi/linux/kfd_ioctl.h> +#include <linux/time.h> +#include "kfd_priv.h" +#include <linux/mm.h> +#include <uapi/asm-generic/mman-common.h> +#include <asm/processor.h> + +/* + * The primary memory I/O features being added for revisions of gfxip + * beyond 7.0 (Kaveri) are: + * + * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b + * + * “Flat” shader memory access – These are new shader vector memory operations + * that do not reference a T#/V# so a “pointer” is what is sourced from the + * vector gprs for direct access to memory. This pointer space has the + * Shared(LDS) and Private(Scratch) memory mapped into this pointer space as + * apertures. The hardware then determines how to direct the memory request + * based on what apertures the request falls in. + * + * Unaligned support and alignment check + * + * + * System Unified Address - SUA + * + * The standard usage for GPU virtual addresses are that they are mapped by + * a set of page tables we call GPUVM and these page tables are managed by + * a combination of vidMM/driver software components. The current virtual + * address (VA) range for GPUVM is 40b. + * + * As of gfxip7.1 and beyond we’re adding the ability for compute memory + * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access + * the same page tables used by host x86 processors and that are managed by + * the operating system. This is via a technique and hardware called ATC/IOMMU. + * The GPU has the capability of accessing both the GPUVM and ATC address + * spaces for a given VMID (process) simultaneously and we call this feature + * system unified address (SUA). + * + * There are three fundamental address modes of operation for a given VMID + * (process) on the GPU: + * + * HSA64 – 64b pointers and the default address space is ATC + * HSA32 – 32b pointers and the default address space is ATC + * GPUVM – 64b pointers and the default address space is GPUVM (driver + * model mode) + * + * + * HSA64 - ATC/IOMMU 64b + * + * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized + * by the CPU so an AMD CPU can only access the high area + * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space + * so the actual VA carried to translation is 48b. There is a “hole” in + * the middle of the 64b VA space. + * + * The GPU not only has access to all of the CPU accessible address space via + * ATC/IOMMU, but it also has access to the GPUVM address space. The “system + * unified address” feature (SUA) is the mapping of GPUVM and ATC address spaces + * into a unified pointer space. The method we take for 64b mode is to map the + * full 40b GPUVM address space into the hole of the 64b address space. + + * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we + * direct requests to be translated via GPUVM page tables instead of the + * IOMMU path. + * + * + * 64b to 49b Address conversion + * + * Note that there are still significant portions of unused regions (holes) + * in the 64b address space even for the GPU. There are several places in + * the pipeline (sw and hw), we wish to compress the 64b virtual address + * to a 49b address. This 49b address is constituted of an “ATC” bit + * plus a 48b virtual address. This 49b address is what is passed to the + * translation hardware. ATC==0 means the 48b address is a GPUVM address + * (max of 2^40 – 1) intended to be translated via GPUVM page tables. + * ATC==1 means the 48b address is intended to be translated via IOMMU + * page tables. + * + * A 64b pointer is compared to the apertures that are defined (Base/Limit), in + * this case the GPUVM aperture (red) is defined and if a pointer falls in this + * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero + * as part of the 64b to 49b conversion. + * + * Where this 64b to 49b conversion is done is a function of the usage. + * Most GPU memory access is via memory objects where the driver builds + * a descriptor which consists of a base address and a memory access by + * the GPU usually consists of some kind of an offset or Cartesian coordinate + * that references this memory descriptor. This is the case for shader + * instructions that reference the T# or V# constants, or for specified + * locations of assets (ex. the shader program location). In these cases + * the driver is what handles the 64b to 49b conversion and the base + * address in the descriptor (ex. V# or T# or shader program location) + * is defined as a 48b address w/ an ATC bit. For this usage a given + * memory object cannot straddle multiple apertures in the 64b address + * space. For example a shader program cannot jump in/out between ATC + * and GPUVM space. + * + * In some cases we wish to pass a 64b pointer to the GPU hardware and + * the GPU hw does the 64b to 49b conversion before passing memory + * requests to the cache/memory system. This is the case for the + * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers + * in scalar and vector GPRs respectively. + * + * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip + * hardware sends a 48b address along w/ an ATC bit, to the memory controller + * on the memory request interfaces. + * + * <client>_MC_rdreq_atc // read request ATC bit + * + * 0 : <client>_MC_rdreq_addr is a GPUVM VA + * + * 1 : <client>_MC_rdreq_addr is a ATC VA + * + * + * “Spare” aperture (APE1) + * + * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use + * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the + * config tables for setting cache policies. The “spare” (APE1) aperture is + * motivated by getting a different Mtype from the default. + * The default aperture isn’t an actual base/limit aperture; it is just the + * address space that doesn’t hit any defined base/limit apertures. + * The following diagram is a complete picture of the gfxip7.x SUA apertures. + * The APE1 can be placed either below or above + * the hole (cannot be in the hole). + * + * + * General Aperture definitions and rules + * + * An aperture register definition consists of a Base, Limit, Mtype, and + * usually an ATC bit indicating which translation tables that aperture uses. + * In all cases (for SUA and DUA apertures discussed later), aperture base + * and limit definitions are 64KB aligned. + * + * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 } + * + * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF } + * + * The base and limit are considered inclusive to an aperture so being inside an + * aperture means (address >= Base) AND (address <= Limit). + * + * In no case is a payload that straddles multiple apertures expected to work. + * For example a load_dword_x4 that starts in one aperture and ends in another, + * does not work. For the vector FLAT_* ops we have detection capability in + * the shader for reporting a “memory violation” back to the SQ block for use + * in traps. A memory violation results when an op falls into the hole, + * or a payload straddles multiple apertures. The S_LOAD instruction + * does not have this detection. + * + * Apertures cannot overlap. + * + * + * + * HSA32 - ATC/IOMMU 32b + * + * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR + * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b + * will not fit so there is only partial visibility to the GPUVM + * space (defined by the aperture) for S_LOAD and FLAT_* ops. + * There is no spare (APE1) aperture for HSA32 mode. + * + * + * GPUVM 64b mode (driver model) + * + * This mode is related to HSA64 in that the difference really is that + * the default aperture is GPUVM (ATC==0) and not ATC space. + * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for + * SUA GPUVM mode, but does not support HSA32/HSA64. + * + * + * Device Unified Address - DUA + * + * Device unified address (DUA) is the name of the feature that maps the + * Shared(LDS) memory and Private(Scratch) memory into the overall address + * space for use by the new FLAT_* vector memory ops. The Shared and + * Private memories are mapped as apertures into the address space, + * and the hardware detects when a FLAT_* memory request is to be redirected + * to the LDS or Scratch memory when it falls into one of these apertures. + * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and + * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes, + * the Shared/Private apertures are always placed in a limited selection of + * options in the hole of the 64b address space. For HSA32 mode, the + * Shared/Private apertures can be placed anywhere in the 32b space except at 0. + * + * + * HSA64 Apertures for FLAT_* vector ops + * + * For HSA64 SUA mode, the Shared and Private apertures are always placed + * in the hole w/ a limited selection of possible locations. The requests + * that fall in the private aperture are expanded as a function of the + * work-item id (tid) and redirected to the location of the + * “hidden private memory”. The hidden private can be placed in either GPUVM + * or ATC space. The addresses that fall in the shared aperture are + * re-directed to the on-chip LDS memory hardware. + * + * + * HSA32 Apertures for FLAT_* vector ops + * + * In HSA32 mode, the Private and Shared apertures can be placed anywhere + * in the 32b space except at 0 (Private or Shared Base at zero disables + * the apertures). If the base address of the apertures are non-zero + * (ie apertures exists), the size is always 64KB. + * + * + * GPUVM Apertures for FLAT_* vector ops + * + * In GPUVM mode, the Shared/Private apertures are specified identically + * to HSA64 mode where they are always in the hole at a limited selection + * of locations. + * + * + * Aperture Definitions for SUA and DUA + * + * The interpretation of the aperture register definitions for a given + * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or + * GPUVM64 discussed in previous sections. The mode is first decoded, and + * then the remaining register decode is a function of the mode. + * + * + * SUA Mode Decode + * + * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from + * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and + * the SH_MEM_CONFIG:PTR32 bits. + * + * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode + * + * 1 0 HSA64 + * + * 1 1 HSA32 + * + * 0 X GPUVM64 + * + * In general the hardware will ignore the PTR32 bit and treat as “0” whenever + * DATA_ATC = “0”, but sw should set PTR32=0 when DATA_ATC=0. + * + * The DATA_ATC bit is only set for compute dispatches. + * All “Draw” dispatches are hardcoded to GPUVM64 mode + * for FLAT_* / S_LOAD operations. + */ + +#define MAKE_GPUVM_APP_BASE(gpu_num) \ + (((uint64_t)(gpu_num) << 61) + 0x1000000000000) + +#define MAKE_GPUVM_APP_LIMIT(base) \ + (((uint64_t)(base) & 0xFFFFFF0000000000) | 0xFFFFFFFFFF) + +#define MAKE_SCRATCH_APP_BASE(gpu_num) \ + (((uint64_t)(gpu_num) << 61) + 0x100000000) + +#define MAKE_SCRATCH_APP_LIMIT(base) \ + (((uint64_t)base & 0xFFFFFFFF00000000) | 0xFFFFFFFF) + +#define MAKE_LDS_APP_BASE(gpu_num) \ + (((uint64_t)(gpu_num) << 61) + 0x0) +#define MAKE_LDS_APP_LIMIT(base) \ + (((uint64_t)(base) & 0xFFFFFFFF00000000) | 0xFFFFFFFF) + +int kfd_init_apertures(struct kfd_process *process) +{ + uint8_t id = 0; + struct kfd_dev *dev; + struct kfd_process_device *pdd; + + mutex_lock(&process->mutex); + + /*Iterating over all devices*/ + while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL && + id < NUM_OF_SUPPORTED_GPUS) { + + pdd = kfd_get_process_device_data(dev, process, 1); + + /* + * For 64 bit process aperture will be statically reserved in + * the x86_64 non canonical process address space + * amdkfd doesn't currently support apertures for 32 bit process + */ + if (process->is_32bit_user_mode) { + pdd->lds_base = pdd->lds_limit = 0; + pdd->gpuvm_base = pdd->gpuvm_limit = 0; + pdd->scratch_base = pdd->scratch_limit = 0; + } else { + /* + * node id couldn't be 0 - the three MSB bits of + * aperture shoudn't be 0 + */ + pdd->lds_base = MAKE_LDS_APP_BASE(id + 1); + + pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); + + pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1); + + pdd->gpuvm_limit = + MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base); + + pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1); + + pdd->scratch_limit = + MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); + } + + dev_dbg(kfd_device, "node id %u\n", id); + dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id); + dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base); + dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit); + dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base); + dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit); + dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base); + dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit); + + id++; + } + + mutex_unlock(&process->mutex); + + return 0; +} + + diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c index f198e5a..c42f53b 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c @@ -38,6 +38,7 @@

static long kfd_ioctl(struct file *, unsigned int, unsigned long); static int kfd_open(struct inode *, struct file *); +static int kfd_mmap(struct file *, struct vm_area_struct *);

static const char kfd_dev_name[] = "kfd";

@@ -46,6 +47,7 @@ static const struct file_operations kfd_fops = { .unlocked_ioctl = kfd_ioctl, .compat_ioctl = kfd_ioctl, .open = kfd_open, + .mmap = kfd_mmap, };

static int kfd_char_dev_major = -1; @@ -96,9 +98,22 @@ struct device *kfd_chardev(void)

static int kfd_open(struct inode *inode, struct file *filep) { + struct kfd_process *process; + if (iminor(inode) != 0) return -ENODEV;

+ process = kfd_create_process(current); + if (IS_ERR(process)) + return PTR_ERR(process); + + process->is_32bit_user_mode = is_compat_task(); + + dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", + process->pasid, process->is_32bit_user_mode); + + kfd_init_apertures(process); + return 0; }

@@ -142,8 +157,9 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, _IOC_NR(cmd), arg);

- /* TODO: add function that retrieves process */ - process = NULL; + process = kfd_get_process(current); + if (IS_ERR(process)) + return PTR_ERR(process);

switch (cmd) { case KFD_IOC_CREATE_QUEUE: @@ -185,3 +201,14 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)

return err; } + +static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct kfd_process *process; + + process = kfd_get_process(current); + if (IS_ERR(process)) + return PTR_ERR(process); + + return kfd_doorbell_mmap(process, vma); +} diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_device.c b/drivers/gpu/drm/radeon/amdkfd/kfd_device.c index 4138694..8e2b075 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_device.c @@ -26,6 +26,8 @@ #include <linux/slab.h> #include "kfd_priv.h"

+#define MQD_SIZE_ALIGNED 768 + static const struct kfd_device_info kaveri_device_info = { .max_pasid_bits = 16, }; @@ -91,6 +93,7 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev) kfd->kgd = kgd; kfd->device_info = device_info; kfd->pdev = pdev; + kfd->init_complete = false;

return kfd; } @@ -98,23 +101,53 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev) bool kgd2kfd_device_init(struct kfd_dev *kfd, const struct kgd2kfd_shared_resources *gpu_resources) { + unsigned int size; + kfd->shared_resources = *gpu_resources;

- if (kfd_topology_add_device(kfd) != 0) - return false; + /* calculate max size of mqds needed for queues */ + size = max_num_of_processes * + max_num_of_queues_per_process * + MQD_SIZE_ALIGNED; + + /* add another 512KB for all other allocations on gart */ + size += 512 * 1024; + + if (kfd2kgd->init_sa_manager(kfd->kgd, size)) { + dev_err(kfd_device, + "Error initializing sa manager for device (%x:%x)\n", + kfd->pdev->vendor, kfd->pdev->device); + goto out; + } + + kfd_doorbell_init(kfd); + + if (kfd_topology_add_device(kfd) != 0) { + dev_err(kfd_device, + "Error adding device (%x:%x) to topology\n", + kfd->pdev->vendor, kfd->pdev->device); + goto kfd_topology_add_device_error; + } +

kfd->init_complete = true; dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor, kfd->pdev->device);

- return true; + goto out; + +kfd_topology_add_device_error: + kfd2kgd->fini_sa_manager(kfd->kgd); + dev_err(kfd_device, + "device (%x:%x) NOT added due to errors\n", + kfd->pdev->vendor, kfd->pdev->device); +out: + return kfd->init_complete; }

void kgd2kfd_device_exit(struct kfd_dev *kfd) { - int err = kfd_topology_remove_device(kfd); - - BUG_ON(err != 0); + kfd_topology_remove_device(kfd);

kfree(kfd); } diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/radeon/amdkfd/kfd_doorbell.c new file mode 100644 index 0000000..541f20e --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_doorbell.c @@ -0,0 +1,236 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "kfd_priv.h" +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/slab.h> + +/* + * This extension supports a kernel level doorbells management for + * the kernel queues. + * Basically the last doorbells page is devoted to kernel queues + * and that's assures that any user process won't get access to the + * kernel doorbells page + */ +static DEFINE_MUTEX(doorbell_mutex); +static unsigned long doorbell_available_index[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)] = { 0 }; +#define KERNEL_DOORBELL_PASID 1 +#define KFD_SIZE_OF_DOORBELL_IN_BYTES 4 + +/* + * Each device exposes a doorbell aperture, a PCI MMIO aperture that + * receives 32-bit writes that are passed to queues as wptr values. + * The doorbells are intended to be written by applications as part + * of queueing work on user-mode queues. + * We assign doorbells to applications in PAGE_SIZE-sized and aligned chunks. + * We map the doorbell address space into user-mode when a process creates + * its first queue on each device. + * Although the mapping is done by KFD, it is equivalent to an mmap of + * the /dev/kfd with the particular device encoded in the mmap offset. + * There will be other uses for mmap of /dev/kfd, so only a range of + * offsets (KFD_MMAP_DOORBELL_START-END) is used for doorbells. + */ + +/* # of doorbell bytes allocated for each process. */ +static inline size_t doorbell_process_allocation(void) +{ + return roundup(KFD_SIZE_OF_DOORBELL_IN_BYTES * + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, + PAGE_SIZE); +} + +/* Doorbell calculations for device init. */ +void kfd_doorbell_init(struct kfd_dev *kfd) +{ + size_t doorbell_start_offset; + size_t doorbell_aperture_size; + size_t doorbell_process_limit; + + /* + * We start with calculations in bytes because the input data might + * only be byte-aligned. + * Only after we have done the rounding can we assume any alignment. + */ + + doorbell_start_offset = roundup(kfd->shared_resources.doorbell_start_offset, + doorbell_process_allocation()); + doorbell_aperture_size = rounddown(kfd->shared_resources.doorbell_aperture_size, + doorbell_process_allocation()); + + if (doorbell_aperture_size > doorbell_start_offset) + doorbell_process_limit = + (doorbell_aperture_size - doorbell_start_offset) / doorbell_process_allocation(); + else + doorbell_process_limit = 0; + + kfd->doorbell_base = kfd->shared_resources.doorbell_physical_address + doorbell_start_offset; + kfd->doorbell_id_offset = doorbell_start_offset / sizeof(u32); + kfd->doorbell_process_limit = doorbell_process_limit - 1; + + kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base, doorbell_process_allocation()); + BUG_ON(!kfd->doorbell_kernel_ptr); + + pr_debug("kfd: doorbell initialization:\n"); + pr_debug("kfd: doorbell base == 0x%08lX\n", + (uintptr_t)kfd->doorbell_base); + + pr_debug("kfd: doorbell_id_offset == 0x%08lX\n", + kfd->doorbell_id_offset); + + pr_debug("kfd: doorbell_process_limit == 0x%08lX\n", + doorbell_process_limit); + + pr_debug("kfd: doorbell_kernel_offset == 0x%08lX\n", + (uintptr_t)kfd->doorbell_base); + + pr_debug("kfd: doorbell aperture size == 0x%08lX\n", + kfd->shared_resources.doorbell_aperture_size); + + pr_debug("kfd: doorbell kernel address == 0x%08lX\n", + (uintptr_t)kfd->doorbell_kernel_ptr); +} + +int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma) +{ + phys_addr_t address; + struct kfd_dev *dev; + + /* + * For simplicitly we only allow mapping of the entire doorbell + * allocation of a single device & process. + */ + if (vma->vm_end - vma->vm_start != doorbell_process_allocation()) + return -EINVAL; + + /* Find kfd device according to gpu id */ + dev = kfd_device_by_id(vma->vm_pgoff); + if (dev == NULL) + return -EINVAL; + + /* Find if pdd exists for combination of process and gpu id */ + if (!kfd_get_process_device_data(dev, process, 0)) + return -EINVAL; + + /* Calculate physical address of doorbell */ + address = kfd_get_process_doorbells(dev, process); + + vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + pr_debug("kfd: mapping doorbell page in kfd_doorbell_mmap\n" + " target user address == 0x%016llX\n" + " physical address == 0x%016llX\n" + " vm_flags == 0x%08lX\n" + " size == 0x%08lX\n", + (long long unsigned int) vma->vm_start, address, vma->vm_flags, + doorbell_process_allocation()); + + + return io_remap_pfn_range(vma, + vma->vm_start, + address >> PAGE_SHIFT, + doorbell_process_allocation(), + vma->vm_page_prot); +} + + +/* get kernel iomem pointer for a doorbell */ +u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, unsigned int *doorbell_off) +{ + u32 inx; + + BUG_ON(!kfd || !doorbell_off); + + mutex_lock(&doorbell_mutex); + inx = find_first_zero_bit(doorbell_available_index, KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); + __set_bit(inx, doorbell_available_index); + mutex_unlock(&doorbell_mutex); + + if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) + return NULL; + + /* caluculating the kernel doorbell offset using "faked" kernel pasid that allocated for kernel queues only */ + *doorbell_off = KERNEL_DOORBELL_PASID * (doorbell_process_allocation()/sizeof(u32)) + inx; + + pr_debug("kfd: get kernel queue doorbell\n" + " doorbell offset == 0x%08d\n" + " kernel address == 0x%08lX\n", + *doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx)); + + return kfd->doorbell_kernel_ptr + inx; +} + +void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr) +{ + unsigned int inx; + + BUG_ON(!kfd || !db_addr); + + inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr); + + mutex_lock(&doorbell_mutex); + __clear_bit(inx, doorbell_available_index); + mutex_unlock(&doorbell_mutex); +} + +inline void write_kernel_doorbell(u32 __iomem *db, u32 value) +{ + if (db) { + writel(value, db); + pr_debug("writing %d to doorbell address 0x%p\n", value, db); + } +} + +/* + * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1 + * to doorbells with the process's doorbell page + */ +unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, + struct kfd_process *process, + unsigned int queue_id) +{ + /* + * doorbell_id_offset accounts for doorbells taken by KGD. + * pasid * doorbell_process_allocation/sizeof(u32) adjusts + * to the process's doorbells + */ + return kfd->doorbell_id_offset + + process->pasid * (doorbell_process_allocation()/sizeof(u32)) + + queue_id; +} + +uint64_t kfd_get_number_elems(struct kfd_dev *kfd) +{ + uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size - + kfd->shared_resources.doorbell_start_offset) / + doorbell_process_allocation() + 1; + + return num_of_elems; + +} + +phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, + struct kfd_process *process) +{ + return dev->doorbell_base + + process->pasid * doorbell_process_allocation(); +} diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_module.c b/drivers/gpu/drm/radeon/amdkfd/kfd_module.c index 0002518..a31bf03 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_module.c +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_module.c @@ -22,7 +22,6 @@

#include <linux/module.h> #include <linux/sched.h> -#include <linux/notifier.h> #include <linux/moduleparam.h> #include <linux/device.h> #include "kfd_priv.h" @@ -46,6 +45,16 @@ static const struct kgd2kfd_calls kgd2kfd = { .resume = kgd2kfd_resume, };

+int max_num_of_processes = KFD_MAX_NUM_OF_PROCESSES_DEFAULT; +module_param(max_num_of_processes, int, 0444); +MODULE_PARM_DESC(max_num_of_processes, + "Kernel cmdline parameter that defines the amdkfd maximum number of supported processes"); + +int max_num_of_queues_per_process = KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT; +module_param(max_num_of_queues_per_process, int, 0444); +MODULE_PARM_DESC(max_num_of_queues_per_process, + "Kernel cmdline parameter that defines the amdkfd maximum number of supported queues per process"); + bool kgd2kfd_init(unsigned interface_version, const struct kfd2kgd_calls *f2g, const struct kgd2kfd_calls **g2f) @@ -69,6 +78,24 @@ static int __init kfd_module_init(void) { int err;

+ /* Verify module parameters */ + if ((max_num_of_processes < 0) || + (max_num_of_processes > KFD_MAX_NUM_OF_PROCESSES)) { + pr_err("kfd: max_num_of_processes must be between 0 to KFD_MAX_NUM_OF_PROCESSES\n"); + return -1; + } + + if ((max_num_of_queues_per_process < 0) || + (max_num_of_queues_per_process > + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)) { + pr_err("kfd: max_num_of_queues_per_process must be between 0 to KFD_MAX_NUM_OF_QUEUES_PER_PROCESS\n"); + return -1; + } + + err = kfd_pasid_init(); + if (err < 0) + goto err_pasid; + err = kfd_chardev_init(); if (err < 0) goto err_ioctl; @@ -84,6 +111,8 @@ static int __init kfd_module_init(void) err_topology: kfd_chardev_exit(); err_ioctl: + kfd_pasid_exit(); +err_pasid: return err; }

@@ -91,6 +120,7 @@ static void __exit kfd_module_exit(void) { kfd_topology_shutdown(); kfd_chardev_exit(); + kfd_pasid_exit(); dev_info(kfd_device, "Removed module\n"); }

diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_pasid.c b/drivers/gpu/drm/radeon/amdkfd/kfd_pasid.c new file mode 100644 index 0000000..bfa5e2d --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_pasid.c @@ -0,0 +1,95 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/slab.h> +#include <linux/types.h> +#include "kfd_priv.h" + +static unsigned long *pasid_bitmap; +static unsigned int pasid_limit; +static DEFINE_MUTEX(pasid_mutex); + +int kfd_pasid_init(void) +{ + pasid_limit = max_num_of_processes; + + pasid_bitmap = kzalloc(DIV_ROUND_UP(pasid_limit, BITS_PER_BYTE), GFP_KERNEL); + if (!pasid_bitmap) + return -ENOMEM; + + set_bit(0, pasid_bitmap); /* PASID 0 is reserved. */ + + return 0; +} + +void kfd_pasid_exit(void) +{ + kfree(pasid_bitmap); +} + +bool kfd_set_pasid_limit(unsigned int new_limit) +{ + if (new_limit < pasid_limit) { + bool ok; + + mutex_lock(&pasid_mutex); + + /* ensure that no pasids >= new_limit are in-use */ + ok = (find_next_bit(pasid_bitmap, pasid_limit, new_limit) == pasid_limit); + if (ok) + pasid_limit = new_limit; + + mutex_unlock(&pasid_mutex); + + return ok; + } + + return true; +} + +inline unsigned int kfd_get_pasid_limit(void) +{ + return pasid_limit; +} + +unsigned int kfd_pasid_alloc(void) +{ + unsigned int found; + + mutex_lock(&pasid_mutex); + + found = find_first_zero_bit(pasid_bitmap, pasid_limit); + if (found == pasid_limit) + found = 0; + else + set_bit(found, pasid_bitmap); + + mutex_unlock(&pasid_mutex); + + return found; +} + +void kfd_pasid_free(unsigned int pasid) +{ + BUG_ON(pasid == 0 || pasid >= pasid_limit); + clear_bit(pasid, pasid_bitmap); +} diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h index 4b7ffb6..43f4bd4 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h @@ -31,6 +31,7 @@ #include <linux/workqueue.h> #include <linux/spinlock.h> #include "../radeon_kfd.h" +#include <linux/kfd_ioctl.h>

#define KFD_SYSFS_FILE_MODE 0444

@@ -40,6 +41,22 @@ /* Macro for allocating structures */ #define kfd_alloc_struct(ptr_to_struct) ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))

+/* Kernel module parameter to specify maximum number of supported processes */ +extern int max_num_of_processes; + +#define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32 +#define KFD_MAX_NUM_OF_PROCESSES 512 + +/* + * Kernel module parameter to specify maximum number of supported queues + * per process + */ +extern int max_num_of_queues_per_process; + +#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128 +#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 + + struct kfd_device_info { const struct kfd_scheduler_class *scheduler_class; unsigned int max_pasid_bits; @@ -54,6 +71,17 @@ struct kfd_dev {

unsigned int id; /* topology stub index */

+ phys_addr_t doorbell_base; /* Start of actual doorbells used by + * KFD. It is aligned for mapping + * into user mode + */ + size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell + * to HW doorbell, GFX reserved some + * at the start) + */ + size_t doorbell_process_limit; /* Number of processes we have doorbell space for. */ + u32 __iomem *doorbell_kernel_ptr; /* this is a pointer for a doorbells page used by kernel queue */ + struct kgd2kfd_shared_resources shared_resources;

bool init_complete; @@ -69,15 +97,121 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd);

extern const struct kfd2kgd_calls *kfd2kgd;

+/* Dummy struct just to make kfd_mem_obj* a unique pointer type. */ +struct kfd_mem_obj { + void *sa_bo; + uint64_t gpu_addr; + uint32_t *ptr; +}; + +enum kfd_mempool { + KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, + KFD_MEMPOOL_FRAMEBUFFER = 3, +}; + /* Character device interface */ int kfd_chardev_init(void); void kfd_chardev_exit(void); struct device *kfd_chardev(void);

+ +/* Data that is per-process-per device. */ +struct kfd_process_device { + /* + * List of all per-device data for a process. + * Starts from kfd_process.per_device_data. + */ + struct list_head per_device_list; + + /* The device that owns this data. */ + struct kfd_dev *dev; + + + /*Apertures*/ + uint64_t lds_base; + uint64_t lds_limit; + uint64_t gpuvm_base; + uint64_t gpuvm_limit; + uint64_t scratch_base; + uint64_t scratch_limit; + + /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ + bool bound; +}; + /* Process data */ struct kfd_process { + /* + * kfd_process are stored in an mm_struct*->kfd_process* + * hash table (kfd_processes in kfd_process.c) + */ + struct hlist_node kfd_processes; + + struct mm_struct *mm; + + struct mutex mutex; + + /* + * In any process, the thread that started main() is the lead + * thread and outlives the rest. + * It is here because amd_iommu_bind_pasid wants a task_struct. + */ + struct task_struct *lead_thread; + + /* We want to receive a notification when the mm_struct is destroyed */ + struct mmu_notifier mmu_notifier; + + /* Use for delayed freeing of kfd_process structure */ + struct rcu_head rcu; + + unsigned int pasid; + + /* + * List of kfd_process_device structures, + * one for each device the process is using. + */ + struct list_head per_device_data; + + /* The process's queues. */ + size_t queue_array_size; + + /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */ + struct kfd_queue **queues; + + unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)]; + + /*Is the user space process 32 bit?*/ + bool is_32bit_user_mode; };

+struct kfd_process *kfd_create_process(const struct task_struct *); +struct kfd_process *kfd_get_process(const struct task_struct *); + +struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, + struct kfd_process *p, + int create_pdd); + +/* PASIDs */ +int kfd_pasid_init(void); +void kfd_pasid_exit(void); +bool kfd_set_pasid_limit(unsigned int new_limit); +unsigned int kfd_get_pasid_limit(void); +unsigned int kfd_pasid_alloc(void); +void kfd_pasid_free(unsigned int pasid); + +/* Doorbells */ +void kfd_doorbell_init(struct kfd_dev *kfd); +int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma); +u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, + unsigned int *doorbell_off); +void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); +u32 read_kernel_doorbell(u32 __iomem *db); +void write_kernel_doorbell(u32 __iomem *db, u32 value); +unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, + struct kfd_process *process, + unsigned int queue_id); + extern struct device *kfd_device;

/* Topology */ @@ -96,4 +230,11 @@ void kgd2kfd_interrupt(struct kfd_dev *dev, const void *ih_ring_entry); void kgd2kfd_suspend(struct kfd_dev *dev); int kgd2kfd_resume(struct kfd_dev *dev);

+/* amdkfd Apertures */ +int kfd_init_apertures(struct kfd_process *process); + +uint64_t kfd_get_number_elems(struct kfd_dev *kfd); +phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, + struct kfd_process *process); + #endif diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_process.c b/drivers/gpu/drm/radeon/amdkfd/kfd_process.c new file mode 100644 index 0000000..fd019c5 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_process.c @@ -0,0 +1,319 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/mutex.h> +#include <linux/log2.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/notifier.h> +struct mm_struct; + +#include "kfd_priv.h" + +/* + * Initial size for the array of queues. + * The allocated size is doubled each time + * it is exceeded up to MAX_PROCESS_QUEUES. + */ +#define INITIAL_QUEUE_ARRAY_SIZE 16 + +/* List of struct kfd_process (field kfd_process). Unique/indexed by mm_struct*. */ +#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ +static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); +static DEFINE_MUTEX(kfd_processes_mutex); + +static struct kfd_process *find_process(const struct task_struct *thread); +static struct kfd_process *create_process(const struct task_struct *thread); + +struct kfd_process *kfd_create_process(const struct task_struct *thread) +{ + struct kfd_process *process; + + if (thread->mm == NULL) + return ERR_PTR(-EINVAL); + + /* Only the pthreads threading model is supported. */ + if (thread->group_leader->mm != thread->mm) + return ERR_PTR(-EINVAL); + + /* Take mmap_sem because we call __mmu_notifier_register inside */ + down_write(&thread->mm->mmap_sem); + + /* + * take kfd processes mutex before starting of process creation + * so there won't be a case where two threads of the same process + * create two kfd_process structures + */ + mutex_lock(&kfd_processes_mutex); + + /* A prior open of /dev/kfd could have already created the process. */ + process = find_process(thread); + if (process) + pr_debug("kfd: process already found\n"); + + if (!process) + process = create_process(thread); + + mutex_unlock(&kfd_processes_mutex); + + up_write(&thread->mm->mmap_sem); + + return process; +} + +struct kfd_process *kfd_get_process(const struct task_struct *thread) +{ + struct kfd_process *process; + + if (thread->mm == NULL) + return ERR_PTR(-EINVAL); + + /* Only the pthreads threading model is supported. */ + if (thread->group_leader->mm != thread->mm) + return ERR_PTR(-EINVAL); + + process = find_process(thread); + + return process; +} + +static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) +{ + struct kfd_process *process; + + hash_for_each_possible_rcu(kfd_processes_table, process, kfd_processes, (uintptr_t)mm) + if (process->mm == mm) + return process; + + return NULL; +} + +static struct kfd_process *find_process(const struct task_struct *thread) +{ + struct kfd_process *p; + + rcu_read_lock(); + p = find_process_by_mm(thread->mm); + rcu_read_unlock(); + + return p; +} + +static void kfd_process_destroy_delayed(struct rcu_head *rcu) +{ + struct kfd_process *p; + + p = container_of(rcu, struct kfd_process, rcu); + BUG_ON(atomic_read(&p->mm->mm_count) <= 0); + + mmdrop(p->mm); + kfree(p); +} + +static void kfd_process_notifier_release(struct mmu_notifier *mn, + struct mm_struct *mm) +{ + struct kfd_process_device *pdd, *temp; + struct kfd_process *p; + + /* + * The kfd_process structure can not be free because the + * mmu_notifier srcu is read locked + */ + p = container_of(mn, struct kfd_process, mmu_notifier); + BUG_ON(p->mm != mm); + + mutex_lock(&kfd_processes_mutex); + hash_del_rcu(&p->kfd_processes); + mutex_unlock(&kfd_processes_mutex); + synchronize_rcu(); + + mutex_lock(&p->mutex); + + list_for_each_entry_safe(pdd, temp, &p->per_device_data, per_device_list) { + list_del(&pdd->per_device_list); + kfree(pdd); + } + + kfd_pasid_free(p->pasid); + + mutex_unlock(&p->mutex); + + mutex_destroy(&p->mutex); + + kfree(p->queues); + + /* + * Because we drop mm_count inside kfd_process_destroy_delayed + * and because the mmu_notifier_unregister function also drop + * mm_count we need to take an extra count here. + */ + atomic_inc(&p->mm->mm_count); + mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm); + mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed); +} + +static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = { + .release = kfd_process_notifier_release, +}; + +static struct kfd_process *create_process(const struct task_struct *thread) +{ + struct kfd_process *process; + int err = -ENOMEM; + + process = kzalloc(sizeof(*process), GFP_KERNEL); + + if (!process) + goto err_alloc_process; + + process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE, sizeof(process->queues[0]), GFP_KERNEL); + if (!process->queues) + goto err_alloc_queues; + + process->pasid = kfd_pasid_alloc(); + if (process->pasid == 0) + goto err_alloc_pasid; + + mutex_init(&process->mutex); + + process->mm = thread->mm; + + /* register notifier */ + process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops; + err = __mmu_notifier_register(&process->mmu_notifier, process->mm); + if (err) + goto err_mmu_notifier; + + hash_add_rcu(kfd_processes_table, &process->kfd_processes, (uintptr_t)process->mm); + + process->lead_thread = thread->group_leader; + + process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE; + + INIT_LIST_HEAD(&process->per_device_data); + + return process; + +err_mmu_notifier: + kfd_pasid_free(process->pasid); +err_alloc_pasid: + kfree(process->queues); +err_alloc_queues: + kfree(process); +err_alloc_process: + return ERR_PTR(err); +} + +struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, + struct kfd_process *p, + int create_pdd) +{ + struct kfd_process_device *pdd = NULL; + + list_for_each_entry(pdd, &p->per_device_data, per_device_list) + if (pdd->dev == dev) + return pdd; + + if (create_pdd) { + pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); + if (pdd != NULL) { + pdd->dev = dev; + list_add(&pdd->per_device_list, &p->per_device_data); + } + } + + return pdd; +} + +/* + * Direct the IOMMU to bind the process (specifically the pasid->mm) to the device. + * Unbinding occurs when the process dies or the device is removed. + * + * Assumes that the process lock is held. + */ +struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, + struct kfd_process *p) +{ + struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1); + + if (pdd == NULL) + return ERR_PTR(-ENOMEM); + + if (pdd->bound) + return pdd; + + pdd->bound = true; + + return pdd; +} + +void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid) +{ + struct kfd_process *p; + struct kfd_process_device *pdd; + int i; + + BUG_ON(dev == NULL); + + rcu_read_lock(); + + hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes) + if (p->pasid == pasid) + break; + + rcu_read_unlock(); + + BUG_ON(p->pasid != pasid); + + mutex_lock(&p->mutex); + + pdd = kfd_get_process_device_data(dev, p, 0); + + /* + * Just mark pdd as unbound, because we still need it to call + * amd_iommu_unbind_pasid() in when the process exits. + * We don't call amd_iommu_unbind_pasid() here + * because the IOMMU called us. + */ + if (pdd) + pdd->bound = false; + + mutex_unlock(&p->mutex); +} + +struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p) +{ + return list_first_entry(&p->per_device_data, struct kfd_process_device, per_device_list); +} + +struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p, struct kfd_process_device *pdd) +{ + if (list_is_last(&pdd->per_device_list, &p->per_device_data)) + return NULL; + return list_next_entry(pdd, per_device_list); +} + +bool kfd_has_process_device_data(struct kfd_process *p) +{ + return !(list_empty(&p->per_device_data)); +} diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c b/drivers/gpu/drm/radeon/radeon_kfd.c index 7b54513..bf3485c 100644 --- a/drivers/gpu/drm/radeon/radeon_kfd.c +++ b/drivers/gpu/drm/radeon/radeon_kfd.c @@ -177,8 +177,7 @@ static u32 pool_to_domain(enum kgd_memory_pool p) static int init_sa_manager(struct kgd_dev *kgd, unsigned int size) { struct radeon_device *rdev = (struct radeon_device *)kgd; - u64 max_offset[4]; - int r, i; + int r;

BUG_ON(kgd == NULL);

@@ -191,21 +190,9 @@ static int init_sa_manager(struct kgd_dev *kgd, unsigned int size) if (r) return r;

- /* Try to pin buffer in first 8MB, 16MB or 64MB of GART */ - max_offset[0] = roundup(size, 8 * 1024 * 1024); - max_offset[1] = roundup(size, 16 * 1024 * 1024); - max_offset[2] = roundup(size, 64 * 1024 * 1024); - max_offset[3] = 0; - - for (i = 0 ; i < 4 ; i++) { - - r = radeon_sa_bo_manager_start(rdev, &rdev->kfd_bo, - max_offset[i]); - if (!r) - return r; - } - - radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo); + r = radeon_sa_bo_manager_start(rdev, &rdev->kfd_bo); + if (r) + radeon_sa_bo_manager_fini(rdev, &rdev->kfd_bo);

return r; }

From: Ben Goz ben.goz@amd.com

The queue module enables allocating and initializing queues uniformly.

v3: remove typedef v3: break pr_debug to one line v3: remove memset v3: add documentation

Signed-off-by: Ben Goz ben.goz@amd.com Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/amdkfd/Makefile | 2 +- drivers/gpu/drm/radeon/amdkfd/kfd_priv.h | 123 +++++++++++++++++++++++++++++- drivers/gpu/drm/radeon/amdkfd/kfd_queue.c | 85 +++++++++++++++++++++ 3 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_queue.c

diff --git a/drivers/gpu/drm/radeon/amdkfd/Makefile b/drivers/gpu/drm/radeon/amdkfd/Makefile index 6d6746e..392728a 100644 --- a/drivers/gpu/drm/radeon/amdkfd/Makefile +++ b/drivers/gpu/drm/radeon/amdkfd/Makefile @@ -6,6 +6,6 @@ ccflags-y := -Iinclude/drm

amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ kfd_pasid.o kfd_doorbell.o kfd_aperture.o \ - kfd_process.o + kfd_process.o kfd_queue.o

obj-$(CONFIG_HSA_RADEON) += amdkfd.o diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h index b55b1cb..cf6d40d 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h @@ -56,7 +56,6 @@ extern int max_num_of_queues_per_process; #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128 #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024

- struct kfd_device_info { const struct kfd_scheduler_class *scheduler_class; unsigned int max_pasid_bits; @@ -116,6 +115,128 @@ void kfd_chardev_exit(void); struct device *kfd_chardev(void);

+/** + * enum kfd_queue_type + * + * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type. + * + * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type. + * + * @KFD_QUEUE_TYPE_HIQ: HIQ queue type. + * + * @KFD_QUEUE_TYPE_DIQ: DIQ queue type. + */ +enum kfd_queue_type { + KFD_QUEUE_TYPE_COMPUTE, + KFD_QUEUE_TYPE_SDMA, + KFD_QUEUE_TYPE_HIQ, + KFD_QUEUE_TYPE_DIQ +}; + +/** + * struct queue_properties + * + * @type: The queue type. + * + * @queue_id: Queue identifier. + * + * @queue_address: Queue ring buffer address. + * + * @queue_size: Queue ring buffer size. + * + * @priority: Defines the queue priority relative to other queues in the process. + * This is just an indication and HW scheduling may override the priority as + * necessary while keeping the relative prioritization. + * the priority granularity is from 0 to f which f is the highest priority. + * currently all queues are initialized with the highest priority. + * + * @queue_percent: This field is partially implemented and currently a zero in + * this field defines that the queue is non active. + * + * @read_ptr: User space address which points to the number of dwords the + * cp read from the ring buffer. This field updates automatically by the H/W. + * + * @write_ptr: Defines the number of dwords written to the ring buffer. + * + * @doorbell_ptr: This field aim is to notify the H/W of new packet written to + * the queue ring buffer. This field should be similar to write_ptr and the user + * should update this field after he updated the write_ptr. + * + * @doorbell_off: The doorbell offset in the doorbell pci-bar. + * + * @is_interop: Defines if this is a interop queue. Interop queue means that the + * queue can access both graphics and compute resources. + * + * @is_active: Defines if the queue is active or not. + * + * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid + * of the queue. + * + * This structure represents the queue properties for each queue no matter if + * it's user mode or kernel mode queue. + * + */ +struct queue_properties { + enum kfd_queue_type type; + unsigned int queue_id; + uint64_t queue_address; + uint64_t queue_size; + uint32_t priority; + uint32_t queue_percent; + uint32_t *read_ptr; + uint32_t *write_ptr; + uint32_t *doorbell_ptr; + uint32_t doorbell_off; + bool is_interop; + bool is_active; + /* Not relevant for user mode queues in cp scheduling */ + unsigned int vmid; +}; + +/** + * struct queue + * + * @list: Queue linked list. + * + * @mqd: The queue MQD. + * + * @mqd_mem_obj: The MQD local gpu memory object. + * + * @gart_mqd_addr: The MQD gart mc address. + * + * @properties: The queue properties. + * + * @mec: Used only in no cp scheduling mode and identifies to micro engine id + * that the queue should be execute on. + * + * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id. + * + * @queue: Used only in no cp scheduliong mode and identifies the queue's slot. + * + * @process: The kfd process that created this queue. + * + * @device: The kfd device that created this queue. + * + * This structure represents user mode compute queues. + * It contains all the necessary data to handle such queues. + * + */ + +struct queue { + struct list_head list; + void *mqd; + struct kfd_mem_obj *mqd_mem_obj; + uint64_t gart_mqd_addr; + struct queue_properties properties; + + uint32_t mec; + uint32_t pipe; + uint32_t queue; + + struct kfd_process *process; + struct kfd_dev *device; +}; + /* Data that is per-process-per device. */ struct kfd_process_device { /* diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_queue.c b/drivers/gpu/drm/radeon/amdkfd/kfd_queue.c new file mode 100644 index 0000000..9a0c90b --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_queue.c @@ -0,0 +1,85 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/slab.h> +#include "kfd_priv.h" + +void print_queue_properties(struct queue_properties *q) +{ + if (!q) + return; + + pr_debug("Printing queue properties:\n"); + pr_debug("Queue Type: %u\n", q->type); + pr_debug("Queue Size: %llu\n", q->queue_size); + pr_debug("Queue percent: %u\n", q->queue_percent); + pr_debug("Queue Address: 0x%llX\n", q->queue_address); + pr_debug("Queue Id: %u\n", q->queue_id); + pr_debug("Queue Process Vmid: %u\n", q->vmid); + pr_debug("Queue Read Pointer: 0x%p\n", q->read_ptr); + pr_debug("Queue Write Pointer: 0x%p\n", q->write_ptr); + pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr); + pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off); +} + +void print_queue(struct queue *q) +{ + if (!q) + return; + pr_debug("Printing queue:\n"); + pr_debug("Queue Type: %u\n", q->properties.type); + pr_debug("Queue Size: %llu\n", q->properties.queue_size); + pr_debug("Queue percent: %u\n", q->properties.queue_percent); + pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address); + pr_debug("Queue Id: %u\n", q->properties.queue_id); + pr_debug("Queue Process Vmid: %u\n", q->properties.vmid); + pr_debug("Queue Read Pointer: 0x%p\n", q->properties.read_ptr); + pr_debug("Queue Write Pointer: 0x%p\n", q->properties.write_ptr); + pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr); + pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off); + pr_debug("Queue MQD Address: 0x%p\n", q->mqd); + pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr); + pr_debug("Queue Process Address: 0x%p\n", q->process); + pr_debug("Queue Device Address: 0x%p\n", q->device); +} + +int init_queue(struct queue **q, struct queue_properties properties) +{ + struct queue *tmp; + + BUG_ON(!q); + + tmp = kzalloc(sizeof(struct queue), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + memcpy(&tmp->properties, &properties, sizeof(struct queue_properties)); + + *q = tmp; + return 0; +} + +void uninit_queue(struct queue *q) +{ + kfree(q); +}

From: Ben Goz ben.goz@amd.com

The kernel queue module enables the amdkfd to establish kernel queues, not exposed to user space.

The kernel queues are used for HIQ (HSA Interface Queue) and DIQ (Debug Interface Queue) operations

v3: remove use of internal typedefs v3: use new gart allocation functions

Signed-off-by: Ben Goz ben.goz@amd.com Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/amdkfd/Makefile | 3 +- .../drm/radeon/amdkfd/kfd_device_queue_manager.h | 101 +++ drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.c | 330 ++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.h | 66 ++ drivers/gpu/drm/radeon/amdkfd/kfd_pm4_headers.h | 682 +++++++++++++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_pm4_opcodes.h | 107 ++++ drivers/gpu/drm/radeon/amdkfd/kfd_priv.h | 33 +- 7 files changed, 1320 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.c create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.h create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_pm4_headers.h create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_pm4_opcodes.h

diff --git a/drivers/gpu/drm/radeon/amdkfd/Makefile b/drivers/gpu/drm/radeon/amdkfd/Makefile index 9f8de8d..020d6c7 100644 --- a/drivers/gpu/drm/radeon/amdkfd/Makefile +++ b/drivers/gpu/drm/radeon/amdkfd/Makefile @@ -6,6 +6,7 @@ ccflags-y := -Iinclude/drm

amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ kfd_pasid.o kfd_doorbell.o kfd_aperture.o \ - kfd_process.o kfd_queue.o kfd_mqd_manager.o + kfd_process.o kfd_queue.o kfd_mqd_manager.o \ + kfd_kernel_queue.o

obj-$(CONFIG_HSA_RADEON) += amdkfd.o diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h new file mode 100644 index 0000000..e3a56ec --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h @@ -0,0 +1,101 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef KFD_DEVICE_QUEUE_MANAGER_H_ +#define KFD_DEVICE_QUEUE_MANAGER_H_ + +#include <linux/rwsem.h> +#include <linux/list.h> +#include "kfd_priv.h" +#include "kfd_mqd_manager.h" + +#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (500) +#define QUEUES_PER_PIPE (8) +#define PIPE_PER_ME_CP_SCHEDULING (3) +#define CIK_VMID_NUM (8) +#define KFD_VMID_START_OFFSET (8) +#define VMID_PER_DEVICE CIK_VMID_NUM +#define KFD_DQM_FIRST_PIPE (0) + +struct device_process_node { + struct qcm_process_device *qpd; + struct list_head list; +}; + +struct device_queue_manager { + int (*create_queue)(struct device_queue_manager *dqm, + struct queue *q, + struct qcm_process_device *qpd, + int *allocate_vmid); + int (*destroy_queue)(struct device_queue_manager *dqm, + struct qcm_process_device *qpd, + struct queue *q); + int (*update_queue)(struct device_queue_manager *dqm, + struct queue *q); + int (*destroy_queues)(struct device_queue_manager *dqm); + struct mqd_manager * (*get_mqd_manager)(struct device_queue_manager *dqm, + enum KFD_MQD_TYPE type); + int (*execute_queues)(struct device_queue_manager *dqm); + int (*register_process)(struct device_queue_manager *dqm, + struct qcm_process_device *qpd); + int (*unregister_process)(struct device_queue_manager *dqm, + struct qcm_process_device *qpd); + int (*initialize)(struct device_queue_manager *dqm); + int (*start)(struct device_queue_manager *dqm); + int (*stop)(struct device_queue_manager *dqm); + void (*uninitialize)(struct device_queue_manager *dqm); + int (*create_kernel_queue)(struct device_queue_manager *dqm, + struct kernel_queue *kq, + struct qcm_process_device *qpd); + void (*destroy_kernel_queue)(struct device_queue_manager *dqm, + struct kernel_queue *kq, + struct qcm_process_device *qpd); + bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, + struct qcm_process_device *qpd, + enum cache_policy default_policy, + enum cache_policy alternate_policy, + void __user *alternate_aperture_base, + uint64_t alternate_aperture_size); + + + struct mqd_manager *mqds[KFD_MQD_TYPE_MAX]; + struct packet_manager packets; + struct kfd_dev *dev; + struct mutex lock; + struct list_head queues; + unsigned int processes_count; + unsigned int queue_count; + unsigned int next_pipe_to_allocate; + unsigned int *allocated_queues; + unsigned int vmid_bitmap; + uint64_t pipelines_addr; + struct kfd_mem_obj *pipeline_mem; + uint64_t fence_gpu_addr; + unsigned int *fence_addr; + struct kfd_mem_obj *fence_mem; + bool active_runlist; +}; + + + +#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.c new file mode 100644 index 0000000..f27f783 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.c @@ -0,0 +1,330 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/types.h> +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/printk.h> +#include "kfd_kernel_queue.h" +#include "kfd_priv.h" +#include "kfd_device_queue_manager.h" +#include "kfd_pm4_headers.h" +#include "kfd_pm4_opcodes.h" + +#define PM4_COUNT_ZERO (((1 << 15) - 1) << 16) + +static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, + enum kfd_queue_type type, unsigned int queue_size) +{ + struct queue_properties prop; + int retval; + union PM4_TYPE_3_HEADER nop; + + BUG_ON(!kq || !dev); + BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ); + + pr_debug("kfd: In func %s initializing queue type %d size %d\n", __func__, KFD_QUEUE_TYPE_HIQ, queue_size); + + nop.opcode = IT_NOP; + nop.type = PM4_TYPE_3; + nop.u32all |= PM4_COUNT_ZERO; + + kq->dev = dev; + kq->nop_packet = nop.u32all; + switch (type) { + case KFD_QUEUE_TYPE_DIQ: + case KFD_QUEUE_TYPE_HIQ: + kq->mqd = dev->dqm->get_mqd_manager(dev->dqm, KFD_MQD_TYPE_CIK_HIQ); + break; + default: + BUG(); + break; + } + + if (kq->mqd == NULL) + return false; + + prop.doorbell_ptr = (uint32_t *)kfd_get_kernel_doorbell(dev, &prop.doorbell_off); + if (prop.doorbell_ptr == NULL) + goto err_get_kernel_doorbell; + + retval = kfd2kgd->allocate_mem(dev->kgd, + queue_size, + PAGE_SIZE, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, + (struct kgd_mem **) &kq->pq); + + if (retval != 0) + goto err_pq_allocate_vidmem; + + kq->pq_kernel_addr = kq->pq->ptr; + kq->pq_gpu_addr = kq->pq->gpu_addr; + + retval = kfd2kgd->allocate_mem(dev->kgd, + sizeof(*kq->rptr_kernel), + 32, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, + (struct kgd_mem **) &kq->rptr_mem); + + if (retval != 0) + goto err_rptr_allocate_vidmem; + + kq->rptr_kernel = kq->rptr_mem->ptr; + kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr; + + retval = kfd2kgd->allocate_mem(dev->kgd, + sizeof(*kq->wptr_kernel), + 32, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, + (struct kgd_mem **) &kq->wptr_mem); + + if (retval != 0) + goto err_wptr_allocate_vidmem; + + kq->wptr_kernel = kq->wptr_mem->ptr; + kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr; + + prop.queue_size = queue_size; + prop.is_interop = false; + prop.priority = 1; + prop.queue_percent = 100; + prop.type = type; + prop.vmid = 0; + prop.queue_address = kq->pq_gpu_addr; + prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr; + prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr; + + if (init_queue(&kq->queue, prop) != 0) + goto err_init_queue; + + kq->queue->device = dev; + kq->queue->process = kfd_get_process(current); + + retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd, &kq->queue->mqd_mem_obj, + &kq->queue->gart_mqd_addr, &kq->queue->properties); + if (retval != 0) + goto err_init_mqd; + + /* assign HIQ to HQD */ + if (type == KFD_QUEUE_TYPE_HIQ) { + pr_debug("assigning hiq to hqd\n"); + kq->queue->pipe = KFD_CIK_HIQ_PIPE; + kq->queue->queue = KFD_CIK_HIQ_QUEUE; + kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe, kq->queue->queue, NULL); + } else { + /* allocate fence for DIQ */ + + retval = kfd2kgd->allocate_mem(dev->kgd, + sizeof(uint32_t), + 32, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, + (struct kgd_mem **) &kq->fence_mem_obj); + + if (retval != 0) + goto err_alloc_fence; + + kq->fence_kernel_address = kq->fence_mem_obj->ptr; + kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr; + } + + print_queue(kq->queue); + + return true; +err_alloc_fence: +err_init_mqd: + uninit_queue(kq->queue); +err_init_queue: + kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->wptr_mem); +err_wptr_allocate_vidmem: + kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->rptr_mem); +err_rptr_allocate_vidmem: + kfd2kgd->free_mem(dev->kgd, (struct kgd_mem *) kq->pq); +err_pq_allocate_vidmem: + pr_err("kfd: error init pq\n"); + kfd_release_kernel_doorbell(dev, (u32 *)prop.doorbell_ptr); +err_get_kernel_doorbell: + pr_err("kfd: error init doorbell"); + return false; + +} + +static void uninitialize(struct kernel_queue *kq) +{ + BUG_ON(!kq); + + if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ) + kq->mqd->destroy_mqd(kq->mqd, + false, + QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, + kq->queue->pipe, + kq->queue->queue); + + kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->rptr_mem); + kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->wptr_mem); + kfd2kgd->free_mem(kq->dev->kgd, (struct kgd_mem *) kq->pq); + kfd_release_kernel_doorbell(kq->dev, (u32 *)kq->queue->properties.doorbell_ptr); + uninit_queue(kq->queue); +} + +static int acquire_packet_buffer(struct kernel_queue *kq, + size_t packet_size_in_dwords, unsigned int **buffer_ptr) +{ + size_t available_size; + size_t queue_size_dwords; + uint32_t wptr, rptr; + unsigned int *queue_address; + + BUG_ON(!kq || !buffer_ptr); + + rptr = *kq->rptr_kernel; + wptr = *kq->wptr_kernel; + queue_address = (unsigned int *)kq->pq_kernel_addr; + queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t); + + pr_debug("kfd: In func %s\nrptr: %d\nwptr: %d\nqueue_address 0x%p\n", __func__, rptr, wptr, queue_address); + + available_size = (rptr - 1 - wptr + queue_size_dwords) % queue_size_dwords; + + if (packet_size_in_dwords >= queue_size_dwords || + packet_size_in_dwords >= available_size) + return -ENOMEM; + + if (wptr + packet_size_in_dwords > queue_size_dwords) { + while (wptr > 0) { + queue_address[wptr] = kq->nop_packet; + wptr = (wptr + 1) % queue_size_dwords; + } + } + + *buffer_ptr = &queue_address[wptr]; + kq->pending_wptr = wptr + packet_size_in_dwords; + + return 0; +} + +static void submit_packet(struct kernel_queue *kq) +{ +#ifdef DEBUG + int i; +#endif + + BUG_ON(!kq); + +#ifdef DEBUG + for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) { + pr_debug("0x%2X ", kq->pq_kernel_addr[i]); + if (i % 15 == 0) + pr_debug("\n"); + } + pr_debug("\n"); +#endif + + *kq->wptr_kernel = kq->pending_wptr; + write_kernel_doorbell((u32 *)kq->queue->properties.doorbell_ptr, kq->pending_wptr); +} + +static int sync_with_hw(struct kernel_queue *kq, unsigned long timeout_ms) +{ + unsigned long org_timeout_ms; + + BUG_ON(!kq); + + org_timeout_ms = timeout_ms; + timeout_ms += jiffies * 1000 / HZ; + while (*kq->wptr_kernel != *kq->rptr_kernel) { + if (time_after(jiffies * 1000 / HZ, timeout_ms)) { + pr_err("kfd: kernel_queue %s timeout expired %lu\n", + __func__, org_timeout_ms); + pr_err("kfd: wptr: %d rptr: %d\n", + *kq->wptr_kernel, *kq->rptr_kernel); + return -ETIME; + } + cpu_relax(); + } + + return 0; +} + +static void rollback_packet(struct kernel_queue *kq) +{ + BUG_ON(!kq); + kq->pending_wptr = *kq->queue->properties.write_ptr; +} + +struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, enum kfd_queue_type type) +{ + struct kernel_queue *kq; + + BUG_ON(!dev); + + kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL); + if (!kq) + return NULL; + + kq->initialize = initialize; + kq->uninitialize = uninitialize; + kq->acquire_packet_buffer = acquire_packet_buffer; + kq->submit_packet = submit_packet; + kq->sync_with_hw = sync_with_hw; + kq->rollback_packet = rollback_packet; + + if (kq->initialize(kq, dev, type, 2048) == false) { + pr_err("kfd: failed to init kernel queue\n"); + kfree(kq); + return NULL; + } + return kq; +} + +void kernel_queue_uninit(struct kernel_queue *kq) +{ + BUG_ON(!kq); + + kq->uninitialize(kq); + kfree(kq); +} + +void test_kq(struct kfd_dev *dev) +{ + struct kernel_queue *kq; + uint32_t *buffer, i; + int retval; + + BUG_ON(!dev); + + pr_debug("kfd: starting kernel queue test\n"); + + kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ); + BUG_ON(!kq); + + retval = kq->acquire_packet_buffer(kq, 5, &buffer); + BUG_ON(retval != 0); + for (i = 0; i < 5; i++) + buffer[i] = kq->nop_packet; + kq->submit_packet(kq); + kq->sync_with_hw(kq, 1000); + + pr_debug("kfd: ending kernel queue test\n"); +} + + diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.h b/drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.h new file mode 100644 index 0000000..e250b0a --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_kernel_queue.h @@ -0,0 +1,66 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef KFD_KERNEL_QUEUE_H_ +#define KFD_KERNEL_QUEUE_H_ + +#include <linux/list.h> +#include <linux/types.h> +#include "kfd_priv.h" + +struct kernel_queue { + /* interface */ + bool (*initialize)(struct kernel_queue *kq, struct kfd_dev *dev, + enum kfd_queue_type type, unsigned int queue_size); + void (*uninitialize)(struct kernel_queue *kq); + int (*acquire_packet_buffer)(struct kernel_queue *kq, + size_t packet_size_in_dwords, unsigned int **buffer_ptr); + void (*submit_packet)(struct kernel_queue *kq); + int (*sync_with_hw)(struct kernel_queue *kq, unsigned long timeout_ms); + void (*rollback_packet)(struct kernel_queue *kq); + + /* data */ + struct kfd_dev *dev; + struct mqd_manager *mqd; + struct queue *queue; + uint32_t pending_wptr; + unsigned int nop_packet; + + struct kfd_mem_obj *rptr_mem; + uint32_t *rptr_kernel; + uint64_t rptr_gpu_addr; + struct kfd_mem_obj *wptr_mem; + uint32_t *wptr_kernel; + uint64_t wptr_gpu_addr; + struct kfd_mem_obj *pq; + uint64_t pq_gpu_addr; + uint32_t *pq_kernel_addr; + + struct kfd_mem_obj *fence_mem_obj; + uint64_t fence_gpu_addr; + void *fence_kernel_address; + + struct list_head list; +}; + +#endif /* KFD_KERNEL_QUEUE_H_ */ diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_pm4_headers.h b/drivers/gpu/drm/radeon/amdkfd/kfd_pm4_headers.h new file mode 100644 index 0000000..95e46f8 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_pm4_headers.h @@ -0,0 +1,682 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef KFD_PM4_HEADERS_H_ +#define KFD_PM4_HEADERS_H_ + +#ifndef PM4_HEADER_DEFINED +#define PM4_HEADER_DEFINED + +union PM4_TYPE_3_HEADER { + struct { + unsigned int predicate:1; /* < 0 for diq packets */ + unsigned int shader_type:1; /* < 0 for diq packets */ + unsigned int reserved1:6; /* < reserved */ + unsigned int opcode:8; /* < IT opcode */ + unsigned int count:14; /* < number of DWORDs - 1 in the information body. */ + unsigned int type:2; /* < packet identifier. It should be 3 for type 3 packets */ + }; + unsigned int u32all; +}; +#endif + +/* + * --------------------_MAP_QUEUES-------------------- + */ + +#ifndef _PM4__MAP_QUEUES_DEFINED +#define _PM4__MAP_QUEUES_DEFINED +enum _map_queues_queue_sel_enum { + queue_sel___map_queues__map_to_specified_queue_slots = 0, + queue_sel___map_queues__map_to_hws_determined_queue_slots = 1, + queue_sel___map_queues__enable_process_queues = 2, + queue_sel___map_queues__reserved = 3 }; + +enum _map_queues_vidmem_enum { + vidmem___map_queues__uses_no_video_memory = 0, + vidmem___map_queues__uses_video_memory = 1 }; + +enum _map_queues_alloc_format_enum { + alloc_format___map_queues__one_per_pipe = 0, + alloc_format___map_queues__all_on_one_pipe = 1 }; + +enum _map_queues_engine_sel_enum { + engine_sel___map_queues__compute = 0, + engine_sel___map_queues__sdma0_queue = 2, + engine_sel___map_queues__sdma1_queue = 3 }; + +struct pm4_map_queues { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int reserved1:4; + enum _map_queues_queue_sel_enum queue_sel:2; + unsigned int reserved2:2; + unsigned int vmid:4; + unsigned int reserved3:4; + enum _map_queues_vidmem_enum vidmem:2; + unsigned int reserved4:6; + enum _map_queues_alloc_format_enum alloc_format:2; + enum _map_queues_engine_sel_enum engine_sel:3; + unsigned int num_queues:3; + } bitfields2; + unsigned int ordinal2; + }; + + struct { + union { + struct { + unsigned int reserved5:2; + unsigned int doorbell_offset:21; + unsigned int reserved6:3; + unsigned int queue:6; + } bitfields3; + unsigned int ordinal3; + }; + + unsigned int mqd_addr_lo; + unsigned int mqd_addr_hi; + unsigned int wptr_addr_lo; + unsigned int wptr_addr_hi; + + } _map_queues_ordinals[1]; /* 1..N of these ordinal groups */ + +}; +#endif + +/* + * --------------------_QUERY_STATUS-------------------- + */ + +#ifndef _PM4__QUERY_STATUS_DEFINED +#define _PM4__QUERY_STATUS_DEFINED +enum _query_status_interrupt_sel_enum { + interrupt_sel___query_status__completion_status = 0, + interrupt_sel___query_status__process_status = 1, + interrupt_sel___query_status__queue_status = 2, + interrupt_sel___query_status__reserved = 3 }; + +enum _query_status_command_enum { + command___query_status__interrupt_only = 0, + command___query_status__fence_only_immediate = 1, + command___query_status__fence_only_after_write_ack = 2, + command___query_status__fence_wait_for_write_ack_send_interrupt = 3 }; + +enum _query_status_engine_sel_enum { + engine_sel___query_status__compute = 0, + engine_sel___query_status__sdma0 = 2, + engine_sel___query_status__sdma1 = 3 }; + +struct pm4_query_status { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int context_id:28; + enum _query_status_interrupt_sel_enum interrupt_sel:2; + enum _query_status_command_enum command:2; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int pasid:16; + unsigned int reserved1:16; + } bitfields3; + struct { + unsigned int reserved2:2; + unsigned int doorbell_offset:21; + unsigned int reserved3:3; + enum _query_status_engine_sel_enum engine_sel:3; + unsigned int reserved4:3; + } bitfields4; + unsigned int ordinal3; + }; + + unsigned int addr_lo; + unsigned int addr_hi; + unsigned int data_lo; + unsigned int data_hi; + +}; +#endif + +/* + * --------------------_UNMAP_QUEUES-------------------- + */ + +#ifndef _PM4__UNMAP_QUEUES_DEFINED +#define _PM4__UNMAP_QUEUES_DEFINED +enum _unmap_queues_action_enum { + action___unmap_queues__preempt_queues = 0, + action___unmap_queues__reset_queues = 1, + action___unmap_queues__disable_process_queues = 2, + action___unmap_queues__reserved = 3 }; + +enum _unmap_queues_queue_sel_enum { + queue_sel___unmap_queues__perform_request_on_specified_queues = 0, + queue_sel___unmap_queues__perform_request_on_pasid_queues = 1, + queue_sel___unmap_queues__perform_request_on_all_active_queues = 2, + queue_sel___unmap_queues__reserved = 3 }; + +enum _unmap_queues_engine_sel_enum { + engine_sel___unmap_queues__compute = 0, + engine_sel___unmap_queues__sdma0 = 2, + engine_sel___unmap_queues__sdma1 = 3 }; + +struct pm4_unmap_queues { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + enum _unmap_queues_action_enum action:2; + unsigned int reserved1:2; + enum _unmap_queues_queue_sel_enum queue_sel:2; + unsigned int reserved2:20; + enum _unmap_queues_engine_sel_enum engine_sel:3; + unsigned int num_queues:3; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int pasid:16; + unsigned int reserved3:16; + } bitfields3; + struct { + unsigned int reserved4:2; + unsigned int doorbell_offset0:21; + unsigned int reserved5:9; + } bitfields4; + unsigned int ordinal3; + }; + + union { + struct { + unsigned int reserved6:2; + unsigned int doorbell_offset1:21; + unsigned int reserved7:9; + } bitfields5; + unsigned int ordinal4; + }; + + union { + struct { + unsigned int reserved8:2; + unsigned int doorbell_offset2:21; + unsigned int reserved9:9; + } bitfields6; + unsigned int ordinal5; + }; + + union { + struct { + unsigned int reserved10:2; + unsigned int doorbell_offset3:21; + unsigned int reserved11:9; + } bitfields7; + unsigned int ordinal6; + }; + +}; +#endif + +/* + * --------------------_SET_RESOURCES-------------------- + */ + +#ifndef _PM4__SET_RESOURCES_DEFINED +#define _PM4__SET_RESOURCES_DEFINED +enum _set_resources_queue_type_enum { + queue_type___set_resources__hsa_interface_queue_hiq = 1, + queue_type___set_resources__hsa_debug_interface_queue = 4 }; + +struct pm4_set_resources { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + + unsigned int vmid_mask:16; + unsigned int unmap_latency:8; + unsigned int reserved1:5; + enum _set_resources_queue_type_enum queue_type:3; + } bitfields2; + unsigned int ordinal2; + }; + + unsigned int queue_mask_lo; + unsigned int queue_mask_hi; + unsigned int gws_mask_lo; + unsigned int gws_mask_hi; + + union { + struct { + unsigned int oac_mask:16; + unsigned int reserved2:16; + } bitfields3; + unsigned int ordinal7; + }; + + union { + struct { + unsigned int gds_heap_base:6; + unsigned int reserved3:5; + unsigned int gds_heap_size:6; + unsigned int reserved4:15; + } bitfields4; + unsigned int ordinal8; + }; + +}; +#endif + +/* + * --------------------_RUN_LIST-------------------- + */ + +#ifndef _PM4__RUN_LIST_DEFINED +#define _PM4__RUN_LIST_DEFINED + +struct pm4_runlist { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int reserved1:2; + unsigned int ib_base_lo:30; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int ib_base_hi:16; + unsigned int reserved2:16; + } bitfields3; + unsigned int ordinal3; + }; + + union { + struct { + unsigned int ib_size:20; + unsigned int chain:1; + unsigned int offload_polling:1; + unsigned int reserved3:1; + unsigned int valid:1; + unsigned int vmid:4; + unsigned int reserved4:4; + } bitfields4; + unsigned int ordinal4; + }; + +}; +#endif + +/* + * --------------------_MAP_PROCESS-------------------- + */ + +#ifndef _PM4__MAP_PROCESS_DEFINED +#define _PM4__MAP_PROCESS_DEFINED + +struct pm4_map_process { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int pasid:16; + unsigned int reserved1:8; + unsigned int diq_enable:1; + unsigned int reserved2:7; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int page_table_base:28; + unsigned int reserved3:4; + } bitfields3; + unsigned int ordinal3; + }; + + unsigned int sh_mem_bases; + unsigned int sh_mem_ape1_base; + unsigned int sh_mem_ape1_limit; + unsigned int sh_mem_config; + unsigned int gds_addr_lo; + unsigned int gds_addr_hi; + + union { + struct { + unsigned int num_gws:6; + unsigned int reserved4:2; + unsigned int num_oac:4; + unsigned int reserved5:4; + unsigned int gds_size:6; + unsigned int reserved6:10; + } bitfields4; + unsigned int ordinal10; + }; + +}; +#endif + +/*--------------------_MAP_QUEUES--------------------*/ + +#ifndef _PM4__MAP_QUEUES_DEFINED +#define _PM4__MAP_QUEUES_DEFINED +enum _MAP_QUEUES_queue_sel_enum { + queue_sel___map_queues__map_to_specified_queue_slots = 0, + queue_sel___map_queues__map_to_hws_determined_queue_slots = 1, + queue_sel___map_queues__enable_process_queues = 2, + queue_sel___map_queues__reserved = 3 }; + +enum _MAP_QUEUES_vidmem_enum { + vidmem___map_queues__uses_no_video_memory = 0, + vidmem___map_queues__uses_video_memory = 1 }; + +enum _MAP_QUEUES_alloc_format_enum { + alloc_format___map_queues__one_per_pipe = 0, + alloc_format___map_queues__all_on_one_pipe = 1 }; + +enum _MAP_QUEUES_engine_sel_enum { + engine_sel___map_queues__compute = 0, + engine_sel___map_queues__sdma0_queue = 2, + engine_sel___map_queues__sdma1_queue = 3 }; + + +struct _PM4__MAP_QUEUES { + union { + PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int reserved1:4; + enum _MAP_QUEUES_queue_sel_enum queue_sel:2; + unsigned int reserved2:2; + unsigned int vmid:4; + unsigned int reserved3:4; + enum _MAP_QUEUES_vidmem_enum vidmem:2; + unsigned int reserved4:6; + enum _MAP_QUEUES_alloc_format_enum alloc_format:2; + enum _MAP_QUEUES_engine_sel_enum engine_sel:3; + unsigned int num_queues:3; + } bitfields2; + unsigned int ordinal2; + }; + + struct { + union { + struct { + unsigned int reserved5:2; + unsigned int doorbell_offset:21; + unsigned int reserved6:3; + unsigned int queue:6; + } bitfields3; + unsigned int ordinal3; + }; + + unsigned int mqd_addr_lo; + + unsigned int mqd_addr_hi; + + unsigned int wptr_addr_lo; + + unsigned int wptr_addr_hi; + + } _map_queues_ordinals[1]; /* 1..N of these ordinal groups */ + +}; +#endif + +/*--------------------_QUERY_STATUS--------------------*/ + +#ifndef _PM4__QUERY_STATUS_DEFINED +#define _PM4__QUERY_STATUS_DEFINED +enum _QUERY_STATUS_interrupt_sel_enum { + interrupt_sel___query_status__completion_status = 0, + interrupt_sel___query_status__process_status = 1, + interrupt_sel___query_status__queue_status = 2, + interrupt_sel___query_status__reserved = 3 }; + +enum _QUERY_STATUS_command_enum { + command___query_status__interrupt_only = 0, + command___query_status__fence_only_immediate = 1, + command___query_status__fence_only_after_write_ack = 2, + command___query_status__fence_wait_for_write_ack_send_interrupt = 3 }; + +enum _QUERY_STATUS_engine_sel_enum { + engine_sel___query_status__compute = 0, + engine_sel___query_status__sdma0 = 2, + engine_sel___query_status__sdma1 = 3 }; + + +struct _PM4__QUERY_STATUS { + union { + PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int context_id:28; + enum _QUERY_STATUS_interrupt_sel_enum interrupt_sel:2; + enum _QUERY_STATUS_command_enum command:2; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int pasid:16; + unsigned int reserved1:16; + } bitfields3; + struct { + unsigned int reserved2:2; + unsigned int doorbell_offset:21; + unsigned int reserved3:3; + enum _QUERY_STATUS_engine_sel_enum engine_sel:3; + unsigned int reserved4:3; + } bitfields4; + unsigned int ordinal3; + }; + + unsigned int addr_lo; + + unsigned int addr_hi; + + unsigned int data_lo; + + unsigned int data_hi; + +}; +#endif + +/* + * --------------------UNMAP_QUEUES-------------------- + */ + +#ifndef _PM4__UNMAP_QUEUES_DEFINED +#define _PM4__UNMAP_QUEUES_DEFINED +enum _unmap_queues_action_enum { + action___unmap_queues__preempt_queues = 0, + action___unmap_queues__reset_queues = 1, + action___unmap_queues__disable_process_queues = 2, + action___unmap_queues__reserved = 3 }; + +enum _unmap_queues_queue_sel_enum { + queue_sel___unmap_queues__perform_request_on_specified_queues = 0, + queue_sel___unmap_queues__perform_request_on_pasid_queues = 1, + queue_sel___unmap_queues__perform_request_on_all_active_queues = 2, + queue_sel___unmap_queues__reserved = 3 }; + +enum _unmap_queues_engine_sel_enum { + engine_sel___unmap_queues__compute = 0, + engine_sel___unmap_queues__sdma0 = 2, + engine_sel___unmap_queues__sdma1 = 3 }; + + +struct pm4_unmap_queues { + union { + PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + _unmap_queues_action_enum action:2; + unsigned int reserved1:2; + + _unmap_queues_queue_sel_enum queue_sel:2; + unsigned int reserved2:20; + + _unmap_queues_engine_sel_enum engine_sel:3; + unsigned int num_queues:3; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int pasid:16; + unsigned int reserved3:16; + } bitfields3; + struct { + unsigned int reserved4:2; + unsigned int doorbell_offset0:21; + unsigned int reserved5:9; + } bitfields4; + unsigned int ordinal3; + }; + + union { + struct { + unsigned int reserved6:2; + unsigned int doorbell_offset1:21; + unsigned int reserved7:9; + } bitfields5; + unsigned int ordinal4; + }; + + union { + struct { + unsigned int reserved8:2; + unsigned int doorbell_offset2:21; + unsigned int reserved9:9; + } bitfields6; + unsigned int ordinal5; + }; + + union { + struct { + unsigned int reserved10:2; + unsigned int doorbell_offset3:21; + unsigned int reserved11:9; + } bitfields7; + unsigned int ordinal6; + }; + +}; +#endif + +/* --------------------_SET_SH_REG--------------------*/ + +#ifndef _PM4__SET_SH_REG_DEFINED +#define _PM4__SET_SH_REG_DEFINED + +struct _PM4__SET_SH_REG { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int reg_offset:16; + unsigned int reserved1:8; + unsigned int vmid_shift:5; + unsigned int insert_vmid:1; + unsigned int reserved2:1; + unsigned int non_incr_addr:1; + } bitfields2; + unsigned int ordinal2; + }; + + unsigned int reg_data[1]; /* 1..N of these fields */ + +}; +#endif + +/*--------------------_SET_CONFIG_REG--------------------*/ + +#ifndef _PM4__SET_CONFIG_REG_DEFINED +#define _PM4__SET_CONFIG_REG_DEFINED + +struct pm4__set_config_reg { + union { + union PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int reg_offset:16; + unsigned int reserved1:8; + unsigned int vmid_shift:5; + unsigned int insert_vmid:1; + unsigned int reserved2:2; + } bitfields2; + unsigned int ordinal2; + }; + + unsigned int reg_data[1]; /* 1..N of these fields */ + +}; +#endif + +#endif /* KFD_PM4_HEADERS_H_ */ diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_pm4_opcodes.h b/drivers/gpu/drm/radeon/amdkfd/kfd_pm4_opcodes.h new file mode 100644 index 0000000..b72fa3b --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_pm4_opcodes.h @@ -0,0 +1,107 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + + +#ifndef KFD_PM4_OPCODES_H +#define KFD_PM4_OPCODES_H + +enum it_opcode_type { + IT_NOP = 0x10, + IT_SET_BASE = 0x11, + IT_CLEAR_STATE = 0x12, + IT_INDEX_BUFFER_SIZE = 0x13, + IT_DISPATCH_DIRECT = 0x15, + IT_DISPATCH_INDIRECT = 0x16, + IT_ATOMIC_GDS = 0x1D, + IT_OCCLUSION_QUERY = 0x1F, + IT_SET_PREDICATION = 0x20, + IT_REG_RMW = 0x21, + IT_COND_EXEC = 0x22, + IT_PRED_EXEC = 0x23, + IT_DRAW_INDIRECT = 0x24, + IT_DRAW_INDEX_INDIRECT = 0x25, + IT_INDEX_BASE = 0x26, + IT_DRAW_INDEX_2 = 0x27, + IT_CONTEXT_CONTROL = 0x28, + IT_INDEX_TYPE = 0x2A, + IT_DRAW_INDIRECT_MULTI = 0x2C, + IT_DRAW_INDEX_AUTO = 0x2D, + IT_NUM_INSTANCES = 0x2F, + IT_DRAW_INDEX_MULTI_AUTO = 0x30, + IT_INDIRECT_BUFFER_CNST = 0x33, + IT_STRMOUT_BUFFER_UPDATE = 0x34, + IT_DRAW_INDEX_OFFSET_2 = 0x35, + IT_DRAW_PREAMBLE = 0x36, + IT_WRITE_DATA = 0x37, + IT_DRAW_INDEX_INDIRECT_MULTI = 0x38, + IT_MEM_SEMAPHORE = 0x39, + IT_COPY_DW = 0x3B, + IT_WAIT_REG_MEM = 0x3C, + IT_INDIRECT_BUFFER = 0x3F, + IT_COPY_DATA = 0x40, + IT_PFP_SYNC_ME = 0x42, + IT_SURFACE_SYNC = 0x43, + IT_COND_WRITE = 0x45, + IT_EVENT_WRITE = 0x46, + IT_EVENT_WRITE_EOP = 0x47, + IT_EVENT_WRITE_EOS = 0x48, + IT_RELEASE_MEM = 0x49, + IT_PREAMBLE_CNTL = 0x4A, + IT_DMA_DATA = 0x50, + IT_ACQUIRE_MEM = 0x58, + IT_REWIND = 0x59, + IT_LOAD_UCONFIG_REG = 0x5E, + IT_LOAD_SH_REG = 0x5F, + IT_LOAD_CONFIG_REG = 0x60, + IT_LOAD_CONTEXT_REG = 0x61, + IT_SET_CONFIG_REG = 0x68, + IT_SET_CONTEXT_REG = 0x69, + IT_SET_CONTEXT_REG_INDIRECT = 0x73, + IT_SET_SH_REG = 0x76, + IT_SET_SH_REG_OFFSET = 0x77, + IT_SET_QUEUE_REG = 0x78, + IT_SET_UCONFIG_REG = 0x79, + IT_SCRATCH_RAM_WRITE = 0x7D, + IT_SCRATCH_RAM_READ = 0x7E, + IT_LOAD_CONST_RAM = 0x80, + IT_WRITE_CONST_RAM = 0x81, + IT_DUMP_CONST_RAM = 0x83, + IT_INCREMENT_CE_COUNTER = 0x84, + IT_INCREMENT_DE_COUNTER = 0x85, + IT_WAIT_ON_CE_COUNTER = 0x86, + IT_WAIT_ON_DE_COUNTER_DIFF = 0x88, + IT_SWITCH_BUFFER = 0x8B, + IT_SET_RESOURCES = 0xA0, + IT_MAP_PROCESS = 0xA1, + IT_MAP_QUEUES = 0xA2, + IT_UNMAP_QUEUES = 0xA3, + IT_QUERY_STATUS = 0xA4, + IT_RUN_LIST = 0xA5, +}; + +#define PM4_TYPE_0 0 +#define PM4_TYPE_2 2 +#define PM4_TYPE_3 3 + +#endif /* KFD_PM4_OPCODES_H */ + diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h index ca8f081..66980df 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h @@ -35,6 +35,15 @@

#define KFD_SYSFS_FILE_MODE 0444

+/* + * When working with cp scheduler we should assign the HIQ manually or via the radeon driver + * to a fixed hqd slot, here are the fixed HIQ hqd slot definitions for Kaveri. + * In Kaveri only the first ME queues participates in the cp scheduling taking that in mind + * we set the HIQ slot in the second ME. + */ +#define KFD_CIK_HIQ_PIPE 4 +#define KFD_CIK_HIQ_QUEUE 0 + /* GPU ID hash width in bits */ #define KFD_GPU_ID_HASH_WIDTH 16

@@ -56,6 +65,11 @@ extern int max_num_of_queues_per_process; #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128 #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024

+enum cache_policy { + cache_policy_coherent, + cache_policy_noncoherent +}; + struct kfd_device_info { const struct kfd_scheduler_class *scheduler_class; unsigned int max_pasid_bits; @@ -83,8 +97,10 @@ struct kfd_dev {

struct kgd2kfd_shared_resources shared_resources;

- bool init_complete; + /* QCM Device instance */ + struct device_queue_manager *dqm;

+ bool init_complete; };

/* KGD2KFD callbacks */ @@ -366,6 +382,21 @@ int kgd2kfd_resume(struct kfd_dev *dev); /* amdkfd Apertures */ int kfd_init_apertures(struct kfd_process *process);

+/* Queue Context Management */ +int init_queue(struct queue **q, struct queue_properties properties); +void uninit_queue(struct queue *q); +void print_queue(struct queue *q); + +/* Packet Manager */ + +struct packet_manager { + struct device_queue_manager *dqm; + struct kernel_queue *priv_queue; + struct mutex lock; + bool allocated; + struct kfd_mem_obj *ib_buffer_obj; +}; + uint64_t kfd_get_number_elems(struct kfd_dev *kfd); phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, struct kfd_process *process);

From: Ben Goz ben.goz@amd.com

The packet manager module builds PM4 packets for the sole use of the CP scheduler. Those packets are used by the HIQ to submit runlists to the CP.

v3: remove include of cik_mqds.h v3: Change lower_32/upper_32 calls to use linux macros v3: use new gart allocation functions v3: add documentation

Signed-off-by: Ben Goz ben.goz@amd.com Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/amdkfd/Makefile | 2 +- drivers/gpu/drm/radeon/amdkfd/kfd_packet_manager.c | 495 +++++++++++++++++++++ drivers/gpu/drm/radeon/amdkfd/kfd_priv.h | 72 +++ 3 files changed, 568 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_packet_manager.c

diff --git a/drivers/gpu/drm/radeon/amdkfd/Makefile b/drivers/gpu/drm/radeon/amdkfd/Makefile index 020d6c7..b88e637 100644 --- a/drivers/gpu/drm/radeon/amdkfd/Makefile +++ b/drivers/gpu/drm/radeon/amdkfd/Makefile @@ -7,6 +7,6 @@ ccflags-y := -Iinclude/drm amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ kfd_pasid.o kfd_doorbell.o kfd_aperture.o \ kfd_process.o kfd_queue.o kfd_mqd_manager.o \ - kfd_kernel_queue.o + kfd_kernel_queue.o kfd_packet_manager.o

obj-$(CONFIG_HSA_RADEON) += amdkfd.o diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/radeon/amdkfd/kfd_packet_manager.c new file mode 100644 index 0000000..aabc17e --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_packet_manager.c @@ -0,0 +1,495 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/slab.h> +#include <linux/mutex.h> +#include "kfd_device_queue_manager.h" +#include "kfd_kernel_queue.h" +#include "kfd_priv.h" +#include "kfd_pm4_headers.h" +#include "kfd_pm4_opcodes.h" + +static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes, unsigned int buffer_size_bytes) +{ + unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t); + + BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes); + *wptr = temp; +} + +static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size) +{ + union PM4_TYPE_3_HEADER header; + + header.u32all = 0; + header.opcode = opcode; + header.count = packet_size/sizeof(uint32_t) - 2; + header.type = PM4_TYPE_3; + + return header.u32all; +} + +static void pm_calc_rlib_size(struct packet_manager *pm, unsigned int *rlib_size, bool *over_subscription) +{ + unsigned int process_count, queue_count; + + BUG_ON(!pm || !rlib_size || !over_subscription); + + process_count = pm->dqm->processes_count; + queue_count = pm->dqm->queue_count; + + /* check if there is over subscription*/ + *over_subscription = false; + if ((process_count >= VMID_PER_DEVICE) || + queue_count > PIPE_PER_ME_CP_SCHEDULING * QUEUES_PER_PIPE) { + *over_subscription = true; + pr_debug("kfd: over subscribed runlist\n"); + } + + /* calculate run list ib allocation size */ + *rlib_size = process_count * sizeof(struct pm4_map_process) + + queue_count * sizeof(struct pm4_map_queues); + + /* increase the allocation size in case we need a chained run list when over subscription */ + if (*over_subscription) + *rlib_size += sizeof(struct pm4_runlist); + + pr_debug("kfd: runlist ib size %d\n", *rlib_size); +} + +static int pm_allocate_runlist_ib(struct packet_manager *pm, unsigned int **rl_buffer, uint64_t *rl_gpu_buffer, + unsigned int *rl_buffer_size, bool *is_over_subscription) +{ + int retval; + + BUG_ON(!pm); + BUG_ON(pm->allocated == true); + BUG_ON(is_over_subscription == NULL); + + pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription); + + retval = kfd2kgd->allocate_mem(pm->dqm->dev->kgd, + *rl_buffer_size, + PAGE_SIZE, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, + (struct kgd_mem **) &pm->ib_buffer_obj); + + if (retval != 0) { + pr_err("kfd: failed to allocate runlist IB\n"); + return retval; + } + + *(void **)rl_buffer = pm->ib_buffer_obj->ptr; + *rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr; + + memset(*rl_buffer, 0, *rl_buffer_size); + pm->allocated = true; + return retval; +} + +static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer, + uint64_t ib, size_t ib_size_in_dwords, bool chain) +{ + struct pm4_runlist *packet; + + BUG_ON(!pm || !buffer || !ib); + + packet = (struct pm4_runlist *)buffer; + + memset(buffer, 0, sizeof(struct pm4_runlist)); + packet->header.u32all = build_pm4_header(IT_RUN_LIST, sizeof(struct pm4_runlist)); + + packet->bitfields4.ib_size = ib_size_in_dwords; + packet->bitfields4.chain = chain ? 1 : 0; + packet->bitfields4.offload_polling = 0; + packet->bitfields4.valid = 1; + packet->bitfields4.vmid = 0; + packet->ordinal2 = lower_32_bits(ib); + packet->bitfields3.ib_base_hi = upper_32_bits(ib); + + return 0; +} + +static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer, struct qcm_process_device *qpd) +{ + struct pm4_map_process *packet; + + BUG_ON(!pm || !buffer || !qpd); + + packet = (struct pm4_map_process *)buffer; + + pr_debug("kfd: In func %s\n", __func__); + + memset(buffer, 0, sizeof(struct pm4_map_process)); + + packet->header.u32all = build_pm4_header(IT_MAP_PROCESS, sizeof(struct pm4_map_process)); + packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0; + packet->bitfields2.pasid = qpd->pqm->process->pasid; + packet->bitfields3.page_table_base = qpd->page_table_base; + packet->bitfields4.gds_size = qpd->gds_size; + packet->bitfields4.num_gws = qpd->num_gws; + packet->bitfields4.num_oac = qpd->num_oac; + + packet->sh_mem_config = qpd->sh_mem_config; + packet->sh_mem_bases = qpd->sh_mem_bases; + packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base; + packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit; + + packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area); + packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area); + + return 0; +} + +static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer, struct queue *q) +{ + struct pm4_map_queues *packet; + + BUG_ON(!pm || !buffer || !q); + + pr_debug("kfd: In func %s\n", __func__); + + packet = (struct pm4_map_queues *)buffer; + memset(buffer, 0, sizeof(struct pm4_map_queues)); + + packet->header.u32all = build_pm4_header(IT_MAP_QUEUES, sizeof(struct pm4_map_queues)); + packet->bitfields2.alloc_format = alloc_format___map_queues__one_per_pipe; + packet->bitfields2.num_queues = 1; + packet->bitfields2.queue_sel = queue_sel___map_queues__map_to_hws_determined_queue_slots; + packet->bitfields2.vidmem = (q->properties.is_interop) ? vidmem___map_queues__uses_video_memory : + vidmem___map_queues__uses_no_video_memory; + + switch (q->properties.type) { + case KFD_QUEUE_TYPE_COMPUTE: + case KFD_QUEUE_TYPE_DIQ: + packet->bitfields2.engine_sel = engine_sel___map_queues__compute; + break; + case KFD_QUEUE_TYPE_SDMA: + packet->bitfields2.engine_sel = engine_sel___map_queues__sdma0_queue; + break; + default: + BUG(); + break; + } + + packet->_map_queues_ordinals[0].bitfields3.doorbell_offset = q->properties.doorbell_off; + packet->_map_queues_ordinals[0].mqd_addr_lo = lower_32_bits(q->gart_mqd_addr); + packet->_map_queues_ordinals[0].mqd_addr_hi = upper_32_bits(q->gart_mqd_addr); + packet->_map_queues_ordinals[0].wptr_addr_lo = lower_32_bits((uint64_t)q->properties.write_ptr); + packet->_map_queues_ordinals[0].wptr_addr_hi = upper_32_bits((uint64_t)q->properties.write_ptr); + + return 0; +} + +static int pm_create_runlist_ib(struct packet_manager *pm, struct list_head *queues, + uint64_t *rl_gpu_addr, size_t *rl_size_bytes) +{ + unsigned int alloc_size_bytes; + unsigned int *rl_buffer, rl_wptr, i; + int retval, proccesses_mapped; + struct device_process_node *cur; + struct qcm_process_device *qpd; + struct queue *q; + struct kernel_queue *kq; + bool is_over_subscription; + + BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr); + + rl_wptr = retval = proccesses_mapped = 0; + + retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr, &alloc_size_bytes, &is_over_subscription); + if (retval != 0) + return retval; + + *rl_size_bytes = alloc_size_bytes; + + pr_debug("kfd: In func %s\n", __func__); + pr_debug("kfd: building runlist ib process count: %d queues count %d\n", pm->dqm->processes_count, + pm->dqm->queue_count); + + /* build the run list ib packet */ + list_for_each_entry(cur, queues, list) { + qpd = cur->qpd; + /* build map process packet */ + if (proccesses_mapped >= pm->dqm->processes_count) { + pr_debug("kfd: not enough space left in runlist IB\n"); + pm_release_ib(pm); + return -ENOMEM; + } + retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd); + if (retval != 0) + return retval; + proccesses_mapped++; + inc_wptr(&rl_wptr, sizeof(struct pm4_map_process), alloc_size_bytes); + list_for_each_entry(kq, &qpd->priv_queue_list, list) { + if (kq->queue->properties.is_active != true) + continue; + retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr], kq->queue); + if (retval != 0) + return retval; + inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), alloc_size_bytes); + } + + list_for_each_entry(q, &qpd->queues_list, list) { + if (q->properties.is_active != true) + continue; + retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr], q); + if (retval != 0) + return retval; + inc_wptr(&rl_wptr, sizeof(struct pm4_map_queues), alloc_size_bytes); + } + } + + pr_debug("kfd: finished map process and queues to runlist\n"); + + if (is_over_subscription) + pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr, alloc_size_bytes / sizeof(uint32_t), true); + + for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++) + pr_debug("0x%2X ", rl_buffer[i]); + pr_debug("\n"); + + return 0; +} + +int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm) +{ + BUG_ON(!dqm); + + pm->dqm = dqm; + mutex_init(&pm->lock); + pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ); + if (pm->priv_queue == NULL) { + mutex_destroy(&pm->lock); + return -ENOMEM; + } + pm->allocated = false; + + return 0; +} + +void pm_uninit(struct packet_manager *pm) +{ + BUG_ON(!pm); + + mutex_destroy(&pm->lock); + kernel_queue_uninit(pm->priv_queue); +} + +int pm_send_set_resources(struct packet_manager *pm, struct scheduling_resources *res) +{ + struct pm4_set_resources *packet; + + BUG_ON(!pm || !res); + + pr_debug("kfd: In func %s\n", __func__); + + mutex_lock(&pm->lock); + pm->priv_queue->acquire_packet_buffer(pm->priv_queue, sizeof(*packet) / sizeof(uint32_t), + (unsigned int **)&packet); + if (packet == NULL) { + mutex_unlock(&pm->lock); + pr_err("kfd: failed to allocate buffer on kernel queue\n"); + return -ENOMEM; + } + + memset(packet, 0, sizeof(struct pm4_set_resources)); + packet->header.u32all = build_pm4_header(IT_SET_RESOURCES, sizeof(struct pm4_set_resources)); + + packet->bitfields2.queue_type = queue_type___set_resources__hsa_interface_queue_hiq; + packet->bitfields2.vmid_mask = res->vmid_mask; + packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY; + packet->bitfields3.oac_mask = res->oac_mask; + packet->bitfields4.gds_heap_base = res->gds_heap_base; + packet->bitfields4.gds_heap_size = res->gds_heap_size; + + packet->gws_mask_lo = lower_32_bits(res->gws_mask); + packet->gws_mask_hi = upper_32_bits(res->gws_mask); + + packet->queue_mask_lo = lower_32_bits(res->queue_mask); + packet->queue_mask_hi = upper_32_bits(res->queue_mask); + + pm->priv_queue->submit_packet(pm->priv_queue); + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); + + mutex_unlock(&pm->lock); + + return 0; +} + +int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues) +{ + uint64_t rl_gpu_ib_addr; + uint32_t *rl_buffer; + size_t rl_ib_size, packet_size_dwords; + int retval; + + BUG_ON(!pm || !dqm_queues); + + retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr, &rl_ib_size); + if (retval != 0) + goto fail_create_runlist_ib; + + pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr); + + packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t); + mutex_lock(&pm->lock); + + retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue, packet_size_dwords, &rl_buffer); + if (retval != 0) + goto fail_acquire_packet_buffer; + + retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr, rl_ib_size / sizeof(uint32_t), false); + if (retval != 0) + goto fail_create_runlist; + + pm->priv_queue->submit_packet(pm->priv_queue); + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); + + mutex_unlock(&pm->lock); + + return retval; + +fail_create_runlist: + pm->priv_queue->rollback_packet(pm->priv_queue); +fail_acquire_packet_buffer: + mutex_unlock(&pm->lock); +fail_create_runlist_ib: + if (pm->allocated == true) + pm_release_ib(pm); + return retval; +} + +int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, uint32_t fence_value) +{ + int retval; + struct pm4_query_status *packet; + + BUG_ON(!pm || !fence_address); + + mutex_lock(&pm->lock); + retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue, + sizeof(struct pm4_query_status) / sizeof(uint32_t), (unsigned int **)&packet); + if (retval != 0) + goto fail_acquire_packet_buffer; + + packet->header.u32all = build_pm4_header(IT_QUERY_STATUS, sizeof(struct pm4_query_status)); + + packet->bitfields2.context_id = 0; + packet->bitfields2.interrupt_sel = interrupt_sel___query_status__completion_status; + packet->bitfields2.command = command___query_status__fence_only_after_write_ack; + + packet->addr_hi = upper_32_bits((uint64_t)fence_address); + packet->addr_lo = lower_32_bits((uint64_t)fence_address); + packet->data_hi = upper_32_bits((uint64_t)fence_value); + packet->data_lo = lower_32_bits((uint64_t)fence_value); + + pm->priv_queue->submit_packet(pm->priv_queue); + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); + mutex_unlock(&pm->lock); + + return 0; + +fail_acquire_packet_buffer: + mutex_unlock(&pm->lock); + return retval; +} + +int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, + enum kfd_preempt_type_filter mode, uint32_t filter_param, bool reset) +{ + int retval; + uint32_t *buffer; + struct pm4_unmap_queues *packet; + + BUG_ON(!pm); + + mutex_lock(&pm->lock); + retval = pm->priv_queue->acquire_packet_buffer(pm->priv_queue, + sizeof(struct pm4_unmap_queues) / sizeof(uint32_t), &buffer); + if (retval != 0) + goto err_acquire_packet_buffer; + + packet = (struct pm4_unmap_queues *)buffer; + memset(buffer, 0, sizeof(struct pm4_unmap_queues)); + + packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES, sizeof(struct pm4_unmap_queues)); + switch (type) { + case KFD_QUEUE_TYPE_COMPUTE: + case KFD_QUEUE_TYPE_DIQ: + packet->bitfields2.engine_sel = engine_sel___unmap_queues__compute; + break; + case KFD_QUEUE_TYPE_SDMA: + packet->bitfields2.engine_sel = engine_sel___unmap_queues__sdma0; + break; + default: + BUG(); + break; + } + + if (reset) + packet->bitfields2.action = action___unmap_queues__reset_queues; + else + packet->bitfields2.action = action___unmap_queues__preempt_queues; + + switch (mode) { + case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: + packet->bitfields2.queue_sel = queue_sel___unmap_queues__perform_request_on_specified_queues; + packet->bitfields2.num_queues = 1; + packet->bitfields4.doorbell_offset0 = filter_param; + break; + case KFD_PRERMPT_TYPE_FILTER_BY_PASID: + packet->bitfields2.queue_sel = queue_sel___unmap_queues__perform_request_on_pasid_queues; + packet->bitfields3.pasid = filter_param; + break; + case KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: + packet->bitfields2.queue_sel = queue_sel___unmap_queues__perform_request_on_all_active_queues; + break; + default: + BUG(); + break; + }; + + pm->priv_queue->submit_packet(pm->priv_queue); + pm->priv_queue->sync_with_hw(pm->priv_queue, KFD_HIQ_TIMEOUT); + + mutex_unlock(&pm->lock); + return 0; + +err_acquire_packet_buffer: + mutex_unlock(&pm->lock); + return retval; +} + +void pm_release_ib(struct packet_manager *pm) +{ + BUG_ON(!pm); + + mutex_lock(&pm->lock); + if (pm->allocated) { + kfd2kgd->free_mem(pm->dqm->dev->kgd, + (struct kgd_mem *) pm->ib_buffer_obj); + pm->allocated = false; + } + mutex_unlock(&pm->lock); +} diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h index 5835d07..34028f8 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h @@ -159,6 +159,21 @@ int kfd_chardev_init(void); void kfd_chardev_exit(void); struct device *kfd_chardev(void);

+/** + * enum kfd_preempt_type_filter + * + * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue. + * + * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the running queues list. + * + * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to specific process. + * + */ +enum kfd_preempt_type_filter { + KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE, + KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES, + KFD_PRERMPT_TYPE_FILTER_BY_PASID +};

/** * enum kfd_queue_type @@ -293,6 +308,51 @@ enum KFD_MQD_TYPE { KFD_MQD_TYPE_MAX };

+struct scheduling_resources { + unsigned int vmid_mask; + enum kfd_queue_type type; + uint64_t queue_mask; + uint64_t gws_mask; + uint32_t oac_mask; + uint32_t gds_heap_base; + uint32_t gds_heap_size; +}; + +struct process_queue_manager { + /* data */ + struct kfd_process *process; + unsigned int num_concurrent_processes; + struct list_head queues; + unsigned long *queue_slot_bitmap; +}; + +struct qcm_process_device { + /* The Device Queue Manager that owns this data */ + struct device_queue_manager *dqm; + struct process_queue_manager *pqm; + /* Device Queue Manager lock */ + struct mutex *lock; + /* Queues list */ + struct list_head queues_list; + struct list_head priv_queue_list; + + unsigned int queue_count; + unsigned int vmid; + bool is_debug; + /* + * All the memory management data should be here too + */ + uint64_t gds_context_area; + uint32_t sh_mem_config; + uint32_t sh_mem_bases; + uint32_t sh_mem_ape1_base; + uint32_t sh_mem_ape1_limit; + uint32_t page_table_base; + uint32_t gds_size; + uint32_t num_gws; + uint32_t num_oac; +}; + /* Data that is per-process-per device. */ struct kfd_process_device { /* @@ -412,12 +472,22 @@ int kgd2kfd_resume(struct kfd_dev *dev); int kfd_init_apertures(struct kfd_process *process);

/* Queue Context Management */ +inline uint32_t lower_32(uint64_t x); +inline uint32_t upper_32(uint64_t x); + int init_queue(struct queue **q, struct queue_properties properties); void uninit_queue(struct queue *q); void print_queue(struct queue *q);

+struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, enum kfd_queue_type type); +void kernel_queue_uninit(struct kernel_queue *kq); + /* Packet Manager */

+#define KFD_HIQ_TIMEOUT (500) + +#define KFD_UNMAP_LATENCY (15) + struct packet_manager { struct device_queue_manager *dqm; struct kernel_queue *priv_queue; @@ -426,6 +496,8 @@ struct packet_manager { struct kfd_mem_obj *ib_buffer_obj; };

+void pm_release_ib(struct packet_manager *pm); + uint64_t kfd_get_number_elems(struct kfd_dev *kfd); phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, struct kfd_process *process);

From: Ben Goz ben.goz@amd.com

The queue scheduler divides into two sections, one section is process bounded and the other section is device bounded. The device bounded section is handled by this module. The DQM module handles queue setup, update and tear-down from the device side. It also supports suspend/resume operation.

v3: change device_init v3: use new gart allocation functions v3: Add documentation

Signed-off-by: Ben Goz ben.goz@amd.com Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/amdkfd/Makefile | 2 +- drivers/gpu/drm/radeon/amdkfd/kfd_device.c | 28 +- .../drm/radeon/amdkfd/kfd_device_queue_manager.c | 989 +++++++++++++++++++++ .../drm/radeon/amdkfd/kfd_device_queue_manager.h | 43 + drivers/gpu/drm/radeon/amdkfd/kfd_priv.h | 13 + 5 files changed, 1073 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.c

diff --git a/drivers/gpu/drm/radeon/amdkfd/Makefile b/drivers/gpu/drm/radeon/amdkfd/Makefile index b18a2b5..e3099c8 100644 --- a/drivers/gpu/drm/radeon/amdkfd/Makefile +++ b/drivers/gpu/drm/radeon/amdkfd/Makefile @@ -8,6 +8,6 @@ amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ kfd_pasid.o kfd_doorbell.o kfd_aperture.o \ kfd_process.o kfd_queue.o kfd_mqd_manager.o \ kfd_kernel_queue.o kfd_packet_manager.o \ - kfd_process_queue_manager.o + kfd_process_queue_manager.o kfd_device_queue_manager.o

obj-$(CONFIG_HSA_RADEON) += amdkfd.o diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_device.c b/drivers/gpu/drm/radeon/amdkfd/kfd_device.c index ce90592..74575de 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_device.c @@ -25,6 +25,7 @@ #include <linux/pci.h> #include <linux/slab.h> #include "kfd_priv.h" +#include "kfd_device_queue_manager.h"

#define MQD_SIZE_ALIGNED 768

@@ -194,12 +195,33 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, } amd_iommu_set_invalidate_ctx_cb(kfd->pdev, iommu_pasid_shutdown_callback);

+ kfd->dqm = device_queue_manager_init(kfd); + if (!kfd->dqm) { + dev_err(kfd_device, + "Error initializing queue manager for device (%x:%x)\n", + kfd->pdev->vendor, kfd->pdev->device); + goto device_queue_manager_error; + } + + if (kfd->dqm->start(kfd->dqm) != 0) { + dev_err(kfd_device, + "Error starting queuen manager for device (%x:%x)\n", + kfd->pdev->vendor, kfd->pdev->device); + goto dqm_start_error; + } + kfd->init_complete = true; dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor, kfd->pdev->device);

+ pr_debug("kfd: Starting kfd with the following scheduling policy %d\n", sched_policy); + goto out;

+dqm_start_error: + device_queue_manager_uninit(kfd->dqm); +device_queue_manager_error: + amd_iommu_free_device(kfd->pdev); device_iommu_pasid_error: kfd_topology_remove_device(kfd); kfd_topology_add_device_error: @@ -214,6 +236,7 @@ out: void kgd2kfd_device_exit(struct kfd_dev *kfd) { if (kfd->init_complete) { + device_queue_manager_uninit(kfd->dqm); amd_iommu_free_device(kfd->pdev); kfd_topology_remove_device(kfd); } @@ -225,8 +248,10 @@ void kgd2kfd_suspend(struct kfd_dev *kfd) { BUG_ON(kfd == NULL);

- if (kfd->init_complete) + if (kfd->init_complete) { + kfd->dqm->stop(kfd->dqm); amd_iommu_free_device(kfd->pdev); + } }

int kgd2kfd_resume(struct kfd_dev *kfd) @@ -243,6 +268,7 @@ int kgd2kfd_resume(struct kfd_dev *kfd) if (err < 0) return -ENXIO; amd_iommu_set_invalidate_ctx_cb(kfd->pdev, iommu_pasid_shutdown_callback); + kfd->dqm->start(kfd->dqm); }

return 0; diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.c new file mode 100644 index 0000000..2c3abd2 --- /dev/null +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.c @@ -0,0 +1,989 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/slab.h> +#include <linux/list.h> +#include <linux/types.h> +#include <linux/printk.h> +#include <linux/bitops.h> +#include "kfd_priv.h" +#include "kfd_device_queue_manager.h" +#include "kfd_mqd_manager.h" +#include "cik_regs.h" +#include "kfd_kernel_queue.h" + +#define CIK_HPD_SIZE_LOG2 11 +#define CIK_HPD_SIZE (1U << CIK_HPD_SIZE_LOG2) + +static bool is_mem_initialized; + +static int init_memory(struct device_queue_manager *dqm); +static int +set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, unsigned int vmid); + +static inline unsigned int get_pipes_num(struct device_queue_manager *dqm) +{ + BUG_ON(!dqm || !dqm->dev); + return dqm->dev->shared_resources.compute_pipe_count; +} + +static inline unsigned int get_first_pipe(struct device_queue_manager *dqm) +{ + BUG_ON(!dqm); + return dqm->dev->shared_resources.first_compute_pipe; +} + +static inline unsigned int get_pipes_num_cpsch(void) +{ + return PIPE_PER_ME_CP_SCHEDULING; +} + +static unsigned int get_sh_mem_bases_nybble_64(struct kfd_process *process, struct kfd_dev *dev) +{ + struct kfd_process_device *pdd; + uint32_t nybble; + + pdd = kfd_get_process_device_data(dev, process, 1); + nybble = (pdd->lds_base >> 60) & 0x0E; + + return nybble; + +} + +static unsigned int get_sh_mem_bases_32(struct kfd_process *process, struct kfd_dev *dev) +{ + struct kfd_process_device *pdd; + unsigned int shared_base; + + pdd = kfd_get_process_device_data(dev, process, 1); + shared_base = (pdd->lds_base >> 16) & 0xFF; + + return shared_base; +} + +static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble); +static void init_process_memory(struct device_queue_manager *dqm, struct qcm_process_device *qpd) +{ + unsigned int temp; + + BUG_ON(!dqm || !qpd); + + /* check if sh_mem_config register already configured */ + if (qpd->sh_mem_config == 0) { + qpd->sh_mem_config = + ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) | + DEFAULT_MTYPE(MTYPE_NONCACHED) | + APE1_MTYPE(MTYPE_NONCACHED); + qpd->sh_mem_ape1_limit = 0; + qpd->sh_mem_ape1_base = 0; + } + + if (qpd->pqm->process->is_32bit_user_mode) { + temp = get_sh_mem_bases_32(qpd->pqm->process, dqm->dev); + qpd->sh_mem_bases = SHARED_BASE(temp); + qpd->sh_mem_config |= PTR32; + } else { + temp = get_sh_mem_bases_nybble_64(qpd->pqm->process, dqm->dev); + qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp); + } + + pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n", + qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases); +} + +static void program_sh_mem_settings(struct device_queue_manager *dqm, struct qcm_process_device *qpd) +{ + return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid, qpd->sh_mem_config, + qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, qpd->sh_mem_bases); +} + +static int create_queue_nocpsch(struct device_queue_manager *dqm, struct queue *q, + struct qcm_process_device *qpd, int *allocate_vmid) +{ + bool set, is_new_vmid; + int bit, retval, pipe, i; + struct mqd_manager *mqd; + + BUG_ON(!dqm || !q || !qpd || !allocate_vmid); + retval = 0; + + pr_debug("kfd: In func %s\n", __func__); + print_queue(q); + + mutex_lock(&dqm->lock); + + if (dqm->vmid_bitmap == 0 && qpd->vmid == 0) { + retval = -ENOMEM; + goto no_vmid; + } + + is_new_vmid = false; + if (qpd->vmid == 0) { + bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); + clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); + + /* Kaveri kfd vmid's strts from vmid 8 */ + *allocate_vmid = qpd->vmid = bit + KFD_VMID_START_OFFSET; + q->properties.vmid = *allocate_vmid; + + + pr_debug("kfd: vmid allocation %d\n", *allocate_vmid); + set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); + qpd->vmid = *allocate_vmid; + is_new_vmid = true; + + program_sh_mem_settings(dqm, qpd); + } + q->properties.vmid = qpd->vmid; + + set = false; + for (i = 0, pipe = dqm->next_pipe_to_allocate; i < get_pipes_num(dqm); + pipe = (pipe + i++) % get_pipes_num(dqm)) { + if (dqm->allocated_queues[pipe] != 0) { + bit = find_first_bit((unsigned long *)&dqm->allocated_queues[pipe], QUEUES_PER_PIPE); + clear_bit(bit, (unsigned long *)&dqm->allocated_queues[pipe]); + q->pipe = pipe; + q->queue = bit; + set = true; + break; + } + } + + if (set == false) { + retval = -EBUSY; + goto no_hqd; + } + pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", + __func__, q->pipe, q->queue); + dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); + + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); + if (mqd == NULL) { + retval = -ENOMEM; + goto fail_get_mqd_manager; + } + + retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, &q->gart_mqd_addr, &q->properties); + if (retval != 0) { + set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); + goto init_mqd_failed; + } + + list_add(&q->list, &qpd->queues_list); + dqm->queue_count++; + + mutex_unlock(&dqm->lock); + return 0; + +init_mqd_failed: +fail_get_mqd_manager: +no_hqd: + if (is_new_vmid == true) { + set_bit(*allocate_vmid - KFD_VMID_START_OFFSET, (unsigned long *)&dqm->vmid_bitmap); + *allocate_vmid = qpd->vmid = q->properties.vmid = 0; + } +no_vmid: + mutex_unlock(&dqm->lock); + return retval; +} + +static int destroy_queue_nocpsch(struct device_queue_manager *dqm, struct qcm_process_device *qpd, struct queue *q) +{ + int retval; + struct mqd_manager *mqd; + + BUG_ON(!dqm || !q || !q->mqd || !qpd); + + retval = 0; + + pr_debug("kfd: In Func %s\n", __func__); + + mutex_lock(&dqm->lock); + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); + if (mqd == NULL) { + retval = -ENOMEM; + goto out; + } + retval = mqd->destroy_mqd(mqd, false, QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, q->pipe, q->queue); + if (retval != 0) + goto out; + + mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); + + set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); + q->queue = q->pipe = 0; + list_del(&q->list); + if (list_empty(&qpd->queues_list)) { + set_bit(qpd->vmid - 8, (unsigned long *)&dqm->vmid_bitmap); + qpd->vmid = 0; + } + dqm->queue_count--; +out: + mutex_unlock(&dqm->lock); + return retval; +} + +static int update_queue_nocpsch(struct device_queue_manager *dqm, struct queue *q) +{ + int retval; + struct mqd_manager *mqd; + + BUG_ON(!dqm || !q || !q->mqd); + + mutex_lock(&dqm->lock); + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); + if (mqd == NULL) { + mutex_unlock(&dqm->lock); + return -ENOMEM; + } + retval = mqd->update_mqd(mqd, q->mqd, &q->properties); + if (q->properties.is_active == true) + dqm->queue_count++; + else + dqm->queue_count--; + + mutex_unlock(&dqm->lock); + return 0; +} + +static int destroy_queues_nocpsch(struct device_queue_manager *dqm) +{ + struct device_process_node *cur; + struct mqd_manager *mqd; + struct queue *q; + + BUG_ON(!dqm); + + mutex_lock(&dqm->lock); + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); + if (mqd == NULL) { + mutex_unlock(&dqm->lock); + return -ENOMEM; + } + + list_for_each_entry(cur, &dqm->queues, list) { + list_for_each_entry(q, &cur->qpd->queues_list, list) { + mqd->destroy_mqd(mqd, false, QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, q->pipe, q->queue); + } + } + + mutex_unlock(&dqm->lock); + + return 0; +} + +static struct mqd_manager *get_mqd_manager_nocpsch(struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) +{ + struct mqd_manager *mqd; + + BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); + + pr_debug("kfd: In func %s mqd type %d\n", __func__, type); + + mqd = dqm->mqds[type]; + if (!mqd) { + mqd = mqd_manager_init(type, dqm->dev); + if (mqd == NULL) + pr_err("kfd: mqd manager is NULL"); + dqm->mqds[type] = mqd; + } + + return mqd; +} + +static int execute_queues_nocpsch(struct device_queue_manager *dqm) +{ + struct qcm_process_device *qpd; + struct device_process_node *node; + struct queue *q; + struct mqd_manager *mqd; + + BUG_ON(!dqm); + + mutex_lock(&dqm->lock); + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); + if (mqd == NULL) { + mutex_unlock(&dqm->lock); + return -ENOMEM; + } + + list_for_each_entry(node, &dqm->queues, list) { + qpd = node->qpd; + list_for_each_entry(q, &qpd->queues_list, list) { + pr_debug("kfd: executing queue (%d, %d)\n", q->pipe, q->queue); + if (mqd->is_occupied(mqd, q->properties.queue_address, q->pipe, q->queue) == false && + q->properties.is_active == true) + mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, q->properties.write_ptr); + } + } + + mutex_unlock(&dqm->lock); + + return 0; +} + +static int register_process_nocpsch(struct device_queue_manager *dqm, struct qcm_process_device *qpd) +{ + struct device_process_node *n; + + BUG_ON(!dqm || !qpd); + + pr_debug("kfd: In func %s\n", __func__); + + n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); + if (!n) + return -ENOMEM; + + n->qpd = qpd; + + mutex_lock(&dqm->lock); + list_add(&n->list, &dqm->queues); + + init_process_memory(dqm, qpd); + dqm->processes_count++; + + mutex_unlock(&dqm->lock); + + return 0; +} + +static int unregister_process_nocpsch(struct device_queue_manager *dqm, struct qcm_process_device *qpd) +{ + int retval; + struct device_process_node *cur, *next; + + BUG_ON(!dqm || !qpd); + + BUG_ON(!list_empty(&qpd->queues_list)); + + pr_debug("kfd: In func %s\n", __func__); + + retval = 0; + mutex_lock(&dqm->lock); + + list_for_each_entry_safe(cur, next, &dqm->queues, list) { + if (qpd == cur->qpd) { + list_del(&cur->list); + dqm->processes_count--; + goto out; + } + } + /* qpd not found in dqm list */ + retval = 1; +out: + mutex_unlock(&dqm->lock); + return retval; +} + +static int +set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, unsigned int vmid) +{ + uint32_t pasid_mapping; + + pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | ATC_VMID_PASID_MAPPING_VALID; + return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping, vmid); +} + +static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble) +{ + /* In 64-bit mode, we can only control the top 3 bits of the LDS, scratch and GPUVM apertures. + * The hardware fills in the remaining 59 bits according to the following pattern: + * LDS: X0000000'00000000 - X0000001'00000000 (4GB) + * Scratch: X0000001'00000000 - X0000002'00000000 (4GB) + * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB) + * + * (where X/Y is the configurable nybble with the low-bit 0) + * + * LDS and scratch will have the same top nybble programmed in the top 3 bits of SH_MEM_BASES.PRIVATE_BASE. + * GPUVM can have a different top nybble programmed in the top 3 bits of SH_MEM_BASES.SHARED_BASE. + * We don't bother to support different top nybbles for LDS/Scratch and GPUVM. + */ + + BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE || top_address_nybble == 0); + + return PRIVATE_BASE(top_address_nybble << 12) | SHARED_BASE(top_address_nybble << 12); +} + +static int init_memory(struct device_queue_manager *dqm) +{ + int i, retval; + + for (i = 8; i < 16; i++) + set_pasid_vmid_mapping(dqm, 0, i); + + retval = kfd2kgd->init_memory(dqm->dev->kgd); + if (retval == 0) + is_mem_initialized = true; + return retval; +} + + +static int init_pipelines(struct device_queue_manager *dqm, unsigned int pipes_num, unsigned int first_pipe) +{ + void *hpdptr; + struct mqd_manager *mqd; + unsigned int i, err, inx; + uint64_t pipe_hpd_addr; + + BUG_ON(!dqm || !dqm->dev); + + pr_debug("kfd: In func %s\n", __func__); + + /* + * Allocate memory for the HPDs. This is hardware-owned per-pipe data. + * The driver never accesses this memory after zeroing it. It doesn't even have + * to be saved/restored on suspend/resume because it contains no data when there + * are no active queues. + */ + + err = kfd2kgd->allocate_mem(dqm->dev->kgd, + CIK_HPD_SIZE * pipes_num, + PAGE_SIZE, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, + (struct kgd_mem **) &dqm->pipeline_mem); + + if (err) { + pr_err("kfd: error allocate vidmem num pipes: %d\n", pipes_num); + return -ENOMEM; + } + + hpdptr = dqm->pipeline_mem->ptr; + dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; + + memset(hpdptr, 0, CIK_HPD_SIZE * pipes_num); + + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE); + if (mqd == NULL) { + kfd2kgd->free_mem(dqm->dev->kgd, + (struct kgd_mem *) dqm->pipeline_mem); + return -ENOMEM; + } + + for (i = 0; i < pipes_num; i++) { + inx = i + first_pipe; + pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_SIZE; + pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); + kfd2kgd->init_pipeline(dqm->dev->kgd, i, CIK_HPD_SIZE_LOG2, pipe_hpd_addr); + } + + return 0; +} + + +static int init_scheduler(struct device_queue_manager *dqm) +{ + int retval; + + BUG_ON(!dqm); + + pr_debug("kfd: In %s\n", __func__); + + retval = init_pipelines(dqm, get_pipes_num(dqm), KFD_DQM_FIRST_PIPE); + if (retval != 0) + return retval; + /* should be later integrated with Evgeny/Alexey memory management code */ + retval = init_memory(dqm); + return retval; +} + +static int initialize_nocpsch(struct device_queue_manager *dqm) +{ + int i; + + BUG_ON(!dqm); + + pr_debug("kfd: In func %s num of pipes: %d\n", __func__, get_pipes_num(dqm)); + + mutex_init(&dqm->lock); + INIT_LIST_HEAD(&dqm->queues); + dqm->queue_count = dqm->next_pipe_to_allocate = 0; + dqm->allocated_queues = kcalloc(get_pipes_num(dqm), sizeof(unsigned int), GFP_KERNEL); + if (!dqm->allocated_queues) { + mutex_destroy(&dqm->lock); + return -ENOMEM; + } + + for (i = 0; i < get_pipes_num(dqm); i++) + dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; + + dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; + + init_scheduler(dqm); + return 0; +} + +static void uninitialize_nocpsch(struct device_queue_manager *dqm) +{ + BUG_ON(!dqm); + + BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); + + kfree(dqm->allocated_queues); + mutex_destroy(&dqm->lock); + kfd2kgd->free_mem(dqm->dev->kgd, + (struct kgd_mem *) dqm->pipeline_mem); +} + +static int start_nocpsch(struct device_queue_manager *dqm) +{ + return 0; +} + +static int stop_nocpsch(struct device_queue_manager *dqm) +{ + return 0; +} + +/* + * Device Queue Manager implementation for cp scheduler + */ + +static int set_sched_resources(struct device_queue_manager *dqm) +{ + struct scheduling_resources res; + unsigned int queue_num, queue_mask; + + BUG_ON(!dqm); + + pr_debug("kfd: In func %s\n", __func__); + + queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; + queue_mask = (1 << queue_num) - 1; + res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; + res.vmid_mask <<= KFD_VMID_START_OFFSET; + res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); + res.gws_mask = res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; + + pr_debug("kfd: scheduling resources:\n" + " vmid mask: 0x%8X\n" + " queue mask: 0x%8llX\n", res.vmid_mask, res.queue_mask); + + return pm_send_set_resources(&dqm->packets, &res); +} + +static int initialize_cpsch(struct device_queue_manager *dqm) +{ + int retval; + + BUG_ON(!dqm); + + pr_debug("kfd: In func %s num of pipes: %d\n", __func__, get_pipes_num_cpsch()); + + mutex_init(&dqm->lock); + INIT_LIST_HEAD(&dqm->queues); + dqm->queue_count = dqm->processes_count = 0; + dqm->active_runlist = false; + retval = init_pipelines(dqm, get_pipes_num(dqm), 0); + if (retval != 0) + goto fail_init_pipelines; + + return 0; + +fail_init_pipelines: + mutex_destroy(&dqm->lock); + return retval; +} + +static int start_cpsch(struct device_queue_manager *dqm) +{ + struct device_process_node *node; + int retval; + + BUG_ON(!dqm); + + retval = 0; + + retval = pm_init(&dqm->packets, dqm); + if (retval != 0) + goto fail_packet_manager_init; + + retval = set_sched_resources(dqm); + if (retval != 0) + goto fail_set_sched_resources; + + pr_debug("kfd: allocating fence memory\n"); + + /* allocate fence memory on the gart */ + retval = kfd2kgd->allocate_mem(dqm->dev->kgd, + sizeof(*dqm->fence_addr), + 32, + KFD_MEMPOOL_SYSTEM_WRITECOMBINE, + (struct kgd_mem **) &dqm->fence_mem); + + if (retval != 0) + goto fail_allocate_vidmem; + + dqm->fence_addr = dqm->fence_mem->ptr; + dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; + + list_for_each_entry(node, &dqm->queues, list) { + if (node->qpd->pqm->process && dqm->dev) + kfd_bind_process_to_device(dqm->dev, node->qpd->pqm->process); + } + + dqm->execute_queues(dqm); + + return 0; +fail_allocate_vidmem: +fail_set_sched_resources: + pm_uninit(&dqm->packets); +fail_packet_manager_init: + return retval; +} + +static int stop_cpsch(struct device_queue_manager *dqm) +{ + struct device_process_node *node; + struct kfd_process_device *pdd; + + BUG_ON(!dqm); + + dqm->destroy_queues(dqm); + + list_for_each_entry(node, &dqm->queues, list) { + pdd = kfd_get_process_device_data(dqm->dev, node->qpd->pqm->process, 1); + pdd->bound = false; + } + kfd2kgd->free_mem(dqm->dev->kgd, + (struct kgd_mem *) dqm->fence_mem); + pm_uninit(&dqm->packets); + + return 0; +} + +static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, + struct kernel_queue *kq, + struct qcm_process_device *qpd) +{ + BUG_ON(!dqm || !kq || !qpd); + + pr_debug("kfd: In func %s\n", __func__); + + mutex_lock(&dqm->lock); + list_add(&kq->list, &qpd->priv_queue_list); + dqm->queue_count++; + qpd->is_debug = true; + mutex_unlock(&dqm->lock); + + return 0; +} + +static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, + struct kernel_queue *kq, + struct qcm_process_device *qpd) +{ + BUG_ON(!dqm || !kq); + + pr_debug("kfd: In %s\n", __func__); + + dqm->destroy_queues(dqm); + + mutex_lock(&dqm->lock); + list_del(&kq->list); + dqm->queue_count--; + qpd->is_debug = false; + mutex_unlock(&dqm->lock); +} + +static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, + struct qcm_process_device *qpd, int *allocate_vmid) +{ + int retval; + struct mqd_manager *mqd; + + BUG_ON(!dqm || !q || !qpd); + + retval = 0; + + if (allocate_vmid) + *allocate_vmid = 0; + + mutex_lock(&dqm->lock); + + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP); + if (mqd == NULL) { + mutex_unlock(&dqm->lock); + return -ENOMEM; + } + + retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, &q->gart_mqd_addr, &q->properties); + if (retval != 0) + goto out; + + list_add(&q->list, &qpd->queues_list); + if (q->properties.is_active) + dqm->queue_count++; + +out: + mutex_unlock(&dqm->lock); + return retval; +} + +int fence_wait_timeout(unsigned int *fence_addr, unsigned int fence_value, unsigned long timeout) +{ + BUG_ON(!fence_addr); + timeout += jiffies; + + while (*fence_addr != fence_value) { + if (time_after(jiffies, timeout)) { + pr_err("kfd: qcm fence wait loop timeout expired\n"); + return -ETIME; + } + cpu_relax(); + } + + return 0; +} + +static int destroy_queues_cpsch(struct device_queue_manager *dqm) +{ + int retval; + + BUG_ON(!dqm); + + retval = 0; + + mutex_lock(&dqm->lock); + if (dqm->active_runlist == false) + goto out; + retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, + KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES, 0, false); + if (retval != 0) + goto out; + + *dqm->fence_addr = KFD_FENCE_INIT; + pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, KFD_FENCE_COMPLETED); + /* should be timed out */ + fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); + pm_release_ib(&dqm->packets); + dqm->active_runlist = false; + +out: + mutex_unlock(&dqm->lock); + return retval; +} + +static int execute_queues_cpsch(struct device_queue_manager *dqm) +{ + int retval; + + BUG_ON(!dqm); + + retval = dqm->destroy_queues(dqm); + if (retval != 0) { + pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccesful queues premption"); + return retval; + } + + if (dqm->queue_count <= 0 || dqm->processes_count <= 0) + return 0; + + mutex_lock(&dqm->lock); + if (dqm->active_runlist) { + retval = 0; + goto out; + } + retval = pm_send_runlist(&dqm->packets, &dqm->queues); + if (retval != 0) { + pr_err("kfd: failed to execute runlist"); + goto out; + } + dqm->active_runlist = true; + +out: + mutex_unlock(&dqm->lock); + return retval; +} + +static int destroy_queue_cpsch(struct device_queue_manager *dqm, struct qcm_process_device *qpd, struct queue *q) +{ + int retval; + struct mqd_manager *mqd; + + BUG_ON(!dqm || !qpd || !q); + + retval = 0; + + /* preempt queues before delete mqd */ + dqm->destroy_queues(dqm); + + mutex_lock(&dqm->lock); + mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP); + if (!mqd) { + retval = -ENOMEM; + goto failed_get_mqd_manager; + } + list_del(&q->list); + + mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); + dqm->queue_count--; + mutex_unlock(&dqm->lock); + + return 0; +failed_get_mqd_manager: + mutex_unlock(&dqm->lock); + return retval; +} + +/* Low bits must be 0000/FFFF as required by HW, high bits must be 0 to stay in user mode. */ +#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL +#define APE1_LIMIT_ALIGNMENT 0xFFFF /* APE1 limit is inclusive and 64K aligned. */ + +static bool set_cache_memory_policy(struct device_queue_manager *dqm, + struct qcm_process_device *qpd, + enum cache_policy default_policy, + enum cache_policy alternate_policy, + void __user *alternate_aperture_base, + uint64_t alternate_aperture_size) +{ + uint32_t default_mtype; + uint32_t ape1_mtype; + + pr_debug("kfd: In func %s\n", __func__); + mutex_lock(&dqm->lock); + + if (alternate_aperture_size == 0) { + /* base > limit disables APE1 */ + qpd->sh_mem_ape1_base = 1; + qpd->sh_mem_ape1_limit = 0; + } else { + /* + * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, SH_MEM_APE1_BASE[31:0], 0x0000 } + * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, SH_MEM_APE1_LIMIT[31:0], 0xFFFF } + * Verify that the base and size parameters can be represented in this format + * and convert them. Additionally restrict APE1 to user-mode addresses. + */ + + uint64_t base = (uintptr_t)alternate_aperture_base; + uint64_t limit = base + alternate_aperture_size - 1; + + if (limit <= base) + goto out; + + if ((base & APE1_FIXED_BITS_MASK) != 0) + goto out; + + if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) + goto out; + + qpd->sh_mem_ape1_base = base >> 16; + qpd->sh_mem_ape1_limit = limit >> 16; + + } + + default_mtype = (default_policy == cache_policy_coherent) ? + MTYPE_NONCACHED : + MTYPE_CACHED; + + ape1_mtype = (alternate_policy == cache_policy_coherent) ? + MTYPE_NONCACHED : + MTYPE_CACHED; + + qpd->sh_mem_config = (qpd->sh_mem_config & PTR32) + | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) + | DEFAULT_MTYPE(default_mtype) + | APE1_MTYPE(ape1_mtype); + + if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) + program_sh_mem_settings(dqm, qpd); + + pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", + qpd->sh_mem_config, qpd->sh_mem_ape1_base, + qpd->sh_mem_ape1_limit); + + mutex_unlock(&dqm->lock); + return true; + +out: + mutex_unlock(&dqm->lock); + return false; +} + +struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) +{ + struct device_queue_manager *dqm; + + BUG_ON(!dev); + + dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); + if (!dqm) + return NULL; + + dqm->dev = dev; + switch (sched_policy) { + case KFD_SCHED_POLICY_HWS: + case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: + /* initialize dqm for cp scheduling */ + dqm->create_queue = create_queue_cpsch; + dqm->initialize = initialize_cpsch; + dqm->start = start_cpsch; + dqm->stop = stop_cpsch; + dqm->destroy_queues = destroy_queues_cpsch; + dqm->execute_queues = execute_queues_cpsch; + dqm->destroy_queue = destroy_queue_cpsch; + dqm->update_queue = update_queue_nocpsch; + dqm->get_mqd_manager = get_mqd_manager_nocpsch; + dqm->register_process = register_process_nocpsch; + dqm->unregister_process = unregister_process_nocpsch; + dqm->uninitialize = uninitialize_nocpsch; + dqm->create_kernel_queue = create_kernel_queue_cpsch; + dqm->destroy_kernel_queue = destroy_kernel_queue_cpsch; + dqm->set_cache_memory_policy = set_cache_memory_policy; + break; + case KFD_SCHED_POLICY_NO_HWS: + /* initialize dqm for no cp scheduling */ + dqm->start = start_nocpsch; + dqm->stop = stop_nocpsch; + dqm->create_queue = create_queue_nocpsch; + dqm->destroy_queue = destroy_queue_nocpsch; + dqm->update_queue = update_queue_nocpsch; + dqm->destroy_queues = destroy_queues_nocpsch; + dqm->get_mqd_manager = get_mqd_manager_nocpsch; + dqm->execute_queues = execute_queues_nocpsch; + dqm->register_process = register_process_nocpsch; + dqm->unregister_process = unregister_process_nocpsch; + dqm->initialize = initialize_nocpsch; + dqm->uninitialize = uninitialize_nocpsch; + dqm->set_cache_memory_policy = set_cache_memory_policy; + break; + default: + BUG(); + break; + } + + if (dqm->initialize(dqm) != 0) { + kfree(dqm); + return NULL; + } + + return dqm; +} + +void device_queue_manager_uninit(struct device_queue_manager *dqm) +{ + BUG_ON(!dqm); + + dqm->uninitialize(dqm); + kfree(dqm); +} + diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h index e3a56ec..82b484c 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_device_queue_manager.h @@ -42,6 +42,49 @@ struct device_process_node { struct list_head list; };

+/** + * struct device_queue_manager + * + * @create_queue: Queue creation routine. + * + * @destroy_queue: Queue destruction routine. + * + * @update_queue: Queue update routine. + * + * @get_mqd_manager: Returns the mqd manager according to the mqd type. + * + * @exeute_queues: Dispatches the queues list to the H/W. + * + * @register_process: This routine associates a specific process with device. + * + * @unregister_process: destroys the associations between process to device. + * + * @initialize: Initializes the pipelines and memory module for that device. + * + * @start: Initializes the resources/modules the the device needs for queues + * execution. This function is called on device initialization and after the + * system woke up after suspension. + * + * @stop: This routine stops execution of all the active queue running on the + * H/W and basically this function called on system suspend. + * + * @uninitialize: Destroys all the device queue manager resources allocated in + * initialize routine. + * + * @create_kernel_queue: Creates kernel queue. Used for debug queue. + * + * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. + * + * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the + * memory apertures. + * + * This struct is a base class for the kfd queues scheduler in the device level. + * The device base class should expose the basic operations for queue creation and + * queue destruction. This base class hides the scheduling mode of the driver and + * the specific implementation of the concrete device. This class is the only class + * in the queues scheduler that configures the H/W. + */ + struct device_queue_manager { int (*create_queue)(struct device_queue_manager *dqm, struct queue *q, diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h index 600d671..75214a1 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h @@ -430,6 +430,7 @@ struct kfd_process { struct kfd_process *kfd_create_process(const struct task_struct *); struct kfd_process *kfd_get_process(const struct task_struct *);

+struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, struct kfd_process *p); void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid); struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, struct kfd_process *p, @@ -485,6 +486,9 @@ void uninit_queue(struct queue *q); void print_queue_properties(struct queue_properties *q); void print_queue(struct queue *q);

+struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, struct kfd_dev *dev); +struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); +void device_queue_manager_uninit(struct device_queue_manager *dqm); struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, enum kfd_queue_type type); void kernel_queue_uninit(struct kernel_queue *kq);

@@ -503,6 +507,8 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);

#define KFD_HIQ_TIMEOUT (500)

+#define KFD_FENCE_COMPLETED (100) +#define KFD_FENCE_INIT (10) #define KFD_UNMAP_LATENCY (15)

struct packet_manager { @@ -513,6 +519,13 @@ struct packet_manager { struct kfd_mem_obj *ib_buffer_obj; };

+int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); +void pm_uninit(struct packet_manager *pm); +int pm_send_set_resources(struct packet_manager *pm, struct scheduling_resources *res); +int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues); +int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, uint32_t fence_value); +int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, + enum kfd_preempt_type_filter mode, uint32_t filter_param, bool reset); void pm_release_ib(struct packet_manager *pm);

uint64_t kfd_get_number_elems(struct kfd_dev *kfd);

From: Ben Goz ben.goz@amd.com

v3: remove use of internal typedefs v3: fix debug prints v3: add checks for parameters v3: use doorbell address from user

Signed-off-by: Ben Goz ben.goz@amd.com Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c | 182 ++++++++++++++++++++- drivers/gpu/drm/radeon/amdkfd/kfd_priv.h | 8 + .../drm/radeon/amdkfd/kfd_process_queue_manager.c | 5 +- include/uapi/linux/kfd_ioctl.h | 7 +- 4 files changed, 196 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c index c42f53b..17725f6 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c @@ -119,17 +119,193 @@ static int kfd_open(struct inode *inode, struct file *filep)

static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, void __user *arg) { - return -ENODEV; + struct kfd_ioctl_create_queue_args args; + struct kfd_dev *dev; + int err = 0; + unsigned int queue_id; + struct kfd_process_device *pdd; + struct queue_properties q_properties; + + memset(&q_properties, 0, sizeof(struct queue_properties)); + + if (copy_from_user(&args, arg, sizeof(args))) + return -EFAULT; + + if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { + pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); + return -EINVAL; + } + + if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) { + pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); + return -EINVAL; + } + + if ((args.ring_base_address) && + (!access_ok(VERIFY_WRITE, args.ring_base_address, sizeof(uint64_t)))) { + pr_err("kfd: can't access ring base address\n"); + return -EFAULT; + } + + if (!is_power_of_2(args.ring_size)) { + pr_err("kfd: ring size must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); + return -EINVAL; + } + + if (!access_ok(VERIFY_WRITE, args.read_pointer_address, sizeof(uint32_t))) { + pr_err("kfd: can't access read pointer\n"); + return -EFAULT; + } + + if (!access_ok(VERIFY_WRITE, args.write_pointer_address, sizeof(uint32_t))) { + pr_err("kfd: can't access write pointer\n"); + return -EFAULT; + } + + q_properties.is_interop = false; + q_properties.queue_percent = args.queue_percentage; + q_properties.priority = args.queue_priority; + q_properties.queue_address = args.ring_base_address; + q_properties.queue_size = args.ring_size; + q_properties.read_ptr = (uint32_t *) args.read_pointer_address; + q_properties.write_ptr = (uint32_t *) args.write_pointer_address; + + + pr_debug("kfd: creating queue ioctl\n"); + + pr_debug("Queue Percentage (%d, %d)\n", + q_properties.queue_percent, args.queue_percentage); + + pr_debug("Queue Priority (%d, %d)\n", + q_properties.priority, args.queue_priority); + + pr_debug("Queue Address (0x%llX, 0x%llX)\n", + q_properties.queue_address, args.ring_base_address); + + pr_debug("Queue Size (0x%llX, %u)\n", + q_properties.queue_size, args.ring_size); + + pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n", + (uint64_t) q_properties.read_ptr, + (uint64_t) q_properties.write_ptr); + + dev = kfd_device_by_id(args.gpu_id); + if (dev == NULL) + return -EINVAL; + + mutex_lock(&p->mutex); + + pdd = kfd_bind_process_to_device(dev, p); + if (IS_ERR(pdd) < 0) { + err = PTR_ERR(pdd); + goto err_bind_process; + } + + pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n", + p->pasid, + dev->id); + + err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 0, KFD_QUEUE_TYPE_COMPUTE, &queue_id); + if (err != 0) + goto err_create_queue; + + args.queue_id = queue_id; + + /* Return gpu_id as doorbell offset for mmap usage */ + args.doorbell_offset = args.gpu_id << PAGE_SHIFT; + + if (copy_to_user(arg, &args, sizeof(args))) { + err = -EFAULT; + goto err_copy_args_out; + } + + mutex_unlock(&p->mutex); + + pr_debug("kfd: queue id %d was created successfully\n", args.queue_id); + + pr_debug("ring buffer address == 0x%016llX\n", + args.ring_base_address); + + pr_debug("read ptr address == 0x%016llX\n", + args.read_pointer_address); + + pr_debug("write ptr address == 0x%016llX\n", + args.write_pointer_address); + + return 0; + +err_copy_args_out: + pqm_destroy_queue(&p->pqm, queue_id); +err_create_queue: +err_bind_process: + mutex_unlock(&p->mutex); + return err; }

static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, void __user *arg) { - return -ENODEV; + int retval; + struct kfd_ioctl_destroy_queue_args args; + + if (copy_from_user(&args, arg, sizeof(args))) + return -EFAULT; + + pr_debug("kfd: destroying queue id %d for PASID %d\n", + args.queue_id, + p->pasid); + + mutex_lock(&p->mutex); + + retval = pqm_destroy_queue(&p->pqm, args.queue_id); + + mutex_unlock(&p->mutex); + return retval; }

static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, void __user *arg) { - return -ENODEV; + int retval; + struct kfd_ioctl_update_queue_args args; + struct queue_properties properties; + + if (copy_from_user(&args, arg, sizeof(args))) + return -EFAULT; + + if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { + pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); + return -EINVAL; + } + + if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) { + pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); + return -EINVAL; + } + + if ((args.ring_base_address) && + (!access_ok(VERIFY_WRITE, args.ring_base_address, sizeof(uint64_t)))) { + pr_err("kfd: can't access ring base address\n"); + return -EFAULT; + } + + if (!is_power_of_2(args.ring_size)) { + pr_err("kfd: ring size must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); + return -EINVAL; + } + + properties.queue_address = args.ring_base_address; + properties.queue_size = args.ring_size; + properties.queue_percent = args.queue_percentage; + properties.priority = args.queue_priority; + + pr_debug("kfd: updating queue id %d for PASID %d\n", args.queue_id, p->pasid); + + mutex_lock(&p->mutex); + + retval = pqm_update_queue(&p->pqm, args.queue_id, &properties); + + mutex_unlock(&p->mutex); + + return retval; }

static long kfd_ioctl_set_memory_policy(struct file *filep, struct kfd_process *p, void __user *arg) diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h index 6c656d8..0e3e18f 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_priv.h @@ -513,7 +513,15 @@ struct process_queue_node {

int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p); void pqm_uninit(struct process_queue_manager *pqm); +int pqm_create_queue(struct process_queue_manager *pqm, + struct kfd_dev *dev, + struct file *f, + struct queue_properties *properties, + unsigned int flags, + enum kfd_queue_type type, + unsigned int *qid); int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid); +int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, struct queue_properties *p);

/* Packet Manager */

diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/radeon/amdkfd/kfd_process_queue_manager.c index 7fc24b3..587b44b 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_process_queue_manager.c @@ -257,7 +257,10 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) pr_debug("kfd: In Func %s\n", __func__);

pqn = get_queue_by_qid(pqm, qid); - BUG_ON(!pqn); + if (pqn == NULL) { + pr_err("kfd: queue id does not match any known queue\n"); + return -EINVAL; + }

dev = NULL; if (pqn->kq) diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index a06e021..1ce8302 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -34,8 +34,11 @@ struct kfd_ioctl_get_version_args { };

/* For kfd_ioctl_create_queue_args.queue_type. */ -#define KFD_IOC_QUEUE_TYPE_COMPUTE 0 -#define KFD_IOC_QUEUE_TYPE_SDMA 1 +#define KFD_IOC_QUEUE_TYPE_COMPUTE 0 +#define KFD_IOC_QUEUE_TYPE_SDMA 1 + +#define KFD_MAX_QUEUE_PERCENTAGE 100 +#define KFD_MAX_QUEUE_PRIORITY 15

struct kfd_ioctl_create_queue_args { uint64_t ring_base_address; /* to KFD */

From: Evgeny Pinchuk evgeny.pinchuk@amd.com

Signed-off-by: Evgeny Pinchuk evgeny.pinchuk@amd.com Signed-off-by: Oded Gabbay oded.gabbay@amd.com --- drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c index cc7ac28..eba5b5d 100644 --- a/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/radeon/amdkfd/kfd_chardev.c @@ -364,7 +364,34 @@ out:

static long kfd_ioctl_get_clock_counters(struct file *filep, struct kfd_process *p, void __user *arg) { - return -ENODEV; + struct kfd_ioctl_get_clock_counters_args args; + struct kfd_dev *dev; + struct timespec time; + + if (copy_from_user(&args, arg, sizeof(args))) + return -EFAULT; + + dev = kfd_device_by_id(args.gpu_id); + if (dev == NULL) + return -EINVAL; + + /* Reading GPU clock counter from KGD */ + args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd); + + /* No access to rdtsc. Using raw monotonic time */ + getrawmonotonic(&time); + args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time); + + get_monotonic_boottime(&time); + args.system_clock_counter = (uint64_t)timespec_to_ns(&time); + + /* Since the counter is in nano-seconds we use 1GHz frequency */ + args.system_clock_freq = 1000000000; + + if (copy_to_user(arg, &args, sizeof(args))) + return -EFAULT; + + return 0; }