From: Ben Goz ben.goz@amd.com
This patch implements the new SDMA interface functions. It also adds defines and structures related to SDMA registers.
Signed-off-by: Ben Goz ben.goz@amd.com --- drivers/gpu/drm/radeon/cik_reg.h | 200 +++++++++++++++++++++++++++++++++++- drivers/gpu/drm/radeon/radeon_kfd.c | 132 +++++++++++++++++++++++- 2 files changed, 329 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/radeon/cik_reg.h b/drivers/gpu/drm/radeon/cik_reg.h index 79c45e8..5008964 100644 --- a/drivers/gpu/drm/radeon/cik_reg.h +++ b/drivers/gpu/drm/radeon/cik_reg.h @@ -147,10 +147,73 @@
#define CIK_LB_DESKTOP_HEIGHT 0x6b0c
+#define KFD_CIK_SDMA_QUEUE_OFFSET 0x200 + +#define SQ_IND_INDEX 0x8DE0 +#define SQ_CMD 0x8DEC +#define SQ_IND_DATA 0x8DE4 + +#define TCP_WATCH0_ADDR_H (0x32A0*4) +#define TCP_WATCH1_ADDR_H (0x32A3*4) +#define TCP_WATCH2_ADDR_H (0x32A6*4) +#define TCP_WATCH3_ADDR_H (0x32A9*4) +#define TCP_WATCH0_ADDR_L (0x32A1*4) +#define TCP_WATCH1_ADDR_L (0x32A4*4) +#define TCP_WATCH2_ADDR_L (0x32A7*4) +#define TCP_WATCH3_ADDR_L (0x32AA*4) +#define TCP_WATCH0_CNTL (0x32A2*4) +#define TCP_WATCH1_CNTL (0x32A5*4) +#define TCP_WATCH2_CNTL (0x32A8*4) +#define TCP_WATCH3_CNTL (0x32AB*4) + #define CP_HQD_IQ_RPTR 0xC970u #define AQL_ENABLE (1U << 0) - -#define IDLE (1 << 2) +#define SDMA0_RLC0_RB_CNTL 0xD400u +#define RB_ENABLE (1 << 0) +#define RB_SIZE(x) (x << 1) +#define RB_SWAP_ENABLE (1 << 9) +#define RPTR_WRITEBACK_ENABLE (1 << 12) +#define RPTR_WRITEBACK_SWAP_ENABLE (1 << 13) +#define RPTR_WRITEBACK_TIMER(x) (x << 16) +#define RB_VMID(x) (x << 24) +#define SDMA0_RLC0_RB_BASE 0xD404u +#define SDMA0_RLC0_RB_BASE_HI 0xD408u +#define SDMA0_RLC0_RB_RPTR 0xD40Cu +#define SDMA0_RLC0_RB_WPTR 0xD410u +#define SDMA0_RLC0_RB_WPTR_POLL_CNTL 0xD414u +#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI 0xD418u +#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO 0xD41Cu +#define SDMA0_RLC0_RB_RPTR_ADDR_HI 0xD420u +#define SDMA0_RLC0_RB_RPTR_ADDR_LO 0xD424u +#define SDMA0_RLC0_IB_CNTL 0xD428u +#define SDMA0_RLC0_IB_RPTR 0xD42Cu +#define SDMA0_RLC0_IB_OFFSET 0xD430u +#define SDMA0_RLC0_IB_BASE_LO 0xD434u +#define SDMA0_RLC0_IB_BASE_HI 0xD438u +#define SDMA0_RLC0_IB_SIZE 0xD43Cu +#define SDMA0_RLC0_SKIP_CNTL 0xD440u +#define SDMA0_RLC0_CONTEXT_STATUS 0xD444u +#define SELECTED (1 << 0) +#define IDLE (1 << 2) +#define EXPIRED (1 << 3) +#define EXCEPTION (1 << 4) +#define CTXSW_ABLE (1 << 7) +#define CTXSW_READY (1 << 8) +#define SDMA0_RLC0_DOORBELL 0xD448u +#define OFFSET(x) (x << 0) +#define DB_ENABLE (1 << 28) +#define CAPTURED (1 << 30) +#define SDMA0_RLC0_VIRTUAL_ADDR 0xD49Cu +#define ATC (1 << 0) +#define VA_PTR32 (1 << 4) +#define VA_SHARED_BASE(x) (x << 8) +#define VM_HOLE (1 << 30) +#define SDMA0_RLC0_APE1_CNTL 0xD4A0u +#define SDMA0_RLC0_DOORBELL_LOG 0xD4A4u +#define SDMA0_RLC0_WATERMARK 0xD4A8u +#define SDMA0_CNTL 0xD010 +#define SDMA1_CNTL 0xD810 +#define AUTO_CTXSW_ENABLE (1 << 18)
struct cik_mqd { uint32_t header; @@ -283,4 +346,137 @@ struct cik_mqd { uint32_t queue_doorbell_id15; };
+struct cik_sdma_rlc_registers { + uint32_t sdma_rlc_rb_cntl; + uint32_t sdma_rlc_rb_base; + uint32_t sdma_rlc_rb_base_hi; + uint32_t sdma_rlc_rb_rptr; + uint32_t sdma_rlc_rb_wptr; + uint32_t sdma_rlc_rb_wptr_poll_cntl; + uint32_t sdma_rlc_rb_wptr_poll_addr_hi; + uint32_t sdma_rlc_rb_wptr_poll_addr_lo; + uint32_t sdma_rlc_rb_rptr_addr_hi; + uint32_t sdma_rlc_rb_rptr_addr_lo; + uint32_t sdma_rlc_ib_cntl; + uint32_t sdma_rlc_ib_rptr; + uint32_t sdma_rlc_ib_offset; + uint32_t sdma_rlc_ib_base_lo; + uint32_t sdma_rlc_ib_base_hi; + uint32_t sdma_rlc_ib_size; + uint32_t sdma_rlc_skip_cntl; + uint32_t sdma_rlc_context_status; + uint32_t sdma_rlc_doorbell; + uint32_t sdma_rlc_virtual_addr; + uint32_t sdma_rlc_ape1_cntl; + uint32_t sdma_rlc_doorbell_log; + uint32_t reserved_22; + uint32_t reserved_23; + uint32_t reserved_24; + uint32_t reserved_25; + uint32_t reserved_26; + uint32_t reserved_27; + uint32_t reserved_28; + uint32_t reserved_29; + uint32_t reserved_30; + uint32_t reserved_31; + uint32_t reserved_32; + uint32_t reserved_33; + uint32_t reserved_34; + uint32_t reserved_35; + uint32_t reserved_36; + uint32_t reserved_37; + uint32_t reserved_38; + uint32_t reserved_39; + uint32_t reserved_40; + uint32_t reserved_41; + uint32_t reserved_42; + uint32_t reserved_43; + uint32_t reserved_44; + uint32_t reserved_45; + uint32_t reserved_46; + uint32_t reserved_47; + uint32_t reserved_48; + uint32_t reserved_49; + uint32_t reserved_50; + uint32_t reserved_51; + uint32_t reserved_52; + uint32_t reserved_53; + uint32_t reserved_54; + uint32_t reserved_55; + uint32_t reserved_56; + uint32_t reserved_57; + uint32_t reserved_58; + uint32_t reserved_59; + uint32_t reserved_60; + uint32_t reserved_61; + uint32_t reserved_62; + uint32_t reserved_63; + uint32_t reserved_64; + uint32_t reserved_65; + uint32_t reserved_66; + uint32_t reserved_67; + uint32_t reserved_68; + uint32_t reserved_69; + uint32_t reserved_70; + uint32_t reserved_71; + uint32_t reserved_72; + uint32_t reserved_73; + uint32_t reserved_74; + uint32_t reserved_75; + uint32_t reserved_76; + uint32_t reserved_77; + uint32_t reserved_78; + uint32_t reserved_79; + uint32_t reserved_80; + uint32_t reserved_81; + uint32_t reserved_82; + uint32_t reserved_83; + uint32_t reserved_84; + uint32_t reserved_85; + uint32_t reserved_86; + uint32_t reserved_87; + uint32_t reserved_88; + uint32_t reserved_89; + uint32_t reserved_90; + uint32_t reserved_91; + uint32_t reserved_92; + uint32_t reserved_93; + uint32_t reserved_94; + uint32_t reserved_95; + uint32_t reserved_96; + uint32_t reserved_97; + uint32_t reserved_98; + uint32_t reserved_99; + uint32_t reserved_100; + uint32_t reserved_101; + uint32_t reserved_102; + uint32_t reserved_103; + uint32_t reserved_104; + uint32_t reserved_105; + uint32_t reserved_106; + uint32_t reserved_107; + uint32_t reserved_108; + uint32_t reserved_109; + uint32_t reserved_110; + uint32_t reserved_111; + uint32_t reserved_112; + uint32_t reserved_113; + uint32_t reserved_114; + uint32_t reserved_115; + uint32_t reserved_116; + uint32_t reserved_117; + uint32_t reserved_118; + uint32_t reserved_119; + uint32_t reserved_120; + uint32_t reserved_121; + uint32_t reserved_122; + uint32_t reserved_123; + uint32_t reserved_124; + uint32_t reserved_125; + uint32_t reserved_126; + uint32_t reserved_127; + uint32_t sdma_engine_id; + uint32_t sdma_queue_id; +}; + #endif diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c b/drivers/gpu/drm/radeon/radeon_kfd.c index 242fd8b..b77aee0 100644 --- a/drivers/gpu/drm/radeon/radeon_kfd.c +++ b/drivers/gpu/drm/radeon/radeon_kfd.c @@ -71,13 +71,17 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, uint32_t queue_id, uint32_t __user *wptr); - +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd); static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address, uint32_t pipe_id, uint32_t queue_id);
static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, unsigned int timeout, uint32_t pipe_id, uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout); +static int kgd_init_sdma_engines(struct kgd_dev *kgd);
static const struct kfd2kgd_calls kfd2kgd = { .init_sa_manager = init_sa_manager, @@ -91,9 +95,13 @@ static const struct kfd2kgd_calls kfd2kgd = { .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, .init_memory = kgd_init_memory, .init_pipeline = kgd_init_pipeline, + .init_sdma_engines = kgd_init_sdma_engines, .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, .hqd_is_occupies = kgd_hqd_is_occupies, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, .get_fw_version = get_fw_version };
@@ -435,11 +443,43 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, return 0; }
+static int kgd_init_sdma_engines(struct kgd_dev *kgd) +{ + uint32_t value; + + value = read_register(kgd, SDMA0_CNTL); + value |= AUTO_CTXSW_ENABLE; + write_register(kgd, SDMA0_CNTL, value); + + value = read_register(kgd, SDMA1_CNTL); + value |= AUTO_CTXSW_ENABLE; + write_register(kgd, SDMA1_CNTL, value); + + return 0; +} + +static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m) +{ + uint32_t retval; + + retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET + + m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; + + pr_debug("kfd: sdma base address: 0x%x\n", retval); + + return retval; +} + static inline struct cik_mqd *get_mqd(void *mqd) { return (struct cik_mqd *)mqd; }
+static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) +{ + return (struct cik_sdma_rlc_registers *)mqd; +} + static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, uint32_t queue_id, uint32_t __user *wptr) { @@ -517,6 +557,45 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, return 0; }
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd) +{ + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_VIRTUAL_ADDR, + m->sdma_rlc_virtual_addr); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_BASE, + m->sdma_rlc_rb_base); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_BASE_HI, + m->sdma_rlc_rb_base_hi); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_LO, + m->sdma_rlc_rb_rptr_addr_lo); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_HI, + m->sdma_rlc_rb_rptr_addr_hi); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_DOORBELL, + m->sdma_rlc_doorbell); + + write_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_CNTL, + m->sdma_rlc_rb_cntl); + + return 0; +} + static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address, uint32_t pipe_id, uint32_t queue_id) { @@ -538,6 +617,24 @@ static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address, return retval; }
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + sdma_rlc_rb_cntl = read_register(kgd, + sdma_base_addr + SDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & RB_ENABLE) + return true; + + return false; +} + static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, unsigned int timeout, uint32_t pipe_id, uint32_t queue_id) @@ -566,6 +663,39 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, return 0; }
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout) +{ + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t temp; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL); + temp = temp & ~RB_ENABLE; + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = read_register(kgd, sdma_base_addr + + SDMA0_RLC0_CONTEXT_STATUS); + if (temp & IDLE) + break; + if (timeout == 0) + return -ETIME; + msleep(20); + timeout -= 20; + } + + write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0); + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0); + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0); + write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0); + + return 0; +} + static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type) { struct radeon_device *rdev = (struct radeon_device *) kgd;