Export dmabuf handles for GTT BOs so that their contents can be accessed using SDMA during checkpoint/restore.
Signed-off-by: David Yat Sin david.yatsin@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 12 ++++++++---- include/uapi/linux/kfd_ioctl.h | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 2c7d76e67ddb..e1e2362841f8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1759,7 +1759,8 @@ static int criu_checkpoint_bos(struct kfd_process *p, goto exit; } } - if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { + if (bo_bucket->alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) { ret = criu_get_prime_handle(&dumper_bo->tbo.base, bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0, @@ -1812,7 +1813,8 @@ static int criu_checkpoint_bos(struct kfd_process *p,
exit: while (ret && bo_index--) { - if (bo_buckets[bo_index].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) + if (bo_buckets[bo_index].alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) close_fd(bo_buckets[bo_index].dmabuf_fd); }
@@ -2211,7 +2213,8 @@ static int criu_restore_bo(struct kfd_process *p,
pr_debug("map memory was successful for the BO\n"); /* create the dmabuf object and export the bo */ - if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { + if (bo_bucket->alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) { ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR, &bo_bucket->dmabuf_fd); if (ret) @@ -2281,7 +2284,8 @@ static int criu_restore_bos(struct kfd_process *p,
exit: while (ret && i--) { - if (bo_buckets[i].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) + if (bo_buckets[i].alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) close_fd(bo_buckets[i].dmabuf_fd); } kvfree(bo_buckets); diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index b40687bf1014..eb9ff85f8556 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -33,9 +33,10 @@ * - 1.5 - Add SVM API * - 1.6 - Query clear flags in SVM get_attr API * - 1.7 - Checkpoint Restore (CRIU) API + * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 7 +#define KFD_IOCTL_MINOR_VERSION 8
struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */
Am 2022-03-08 um 16:08 schrieb David Yat Sin:
Export dmabuf handles for GTT BOs so that their contents can be accessed using SDMA during checkpoint/restore.
Signed-off-by: David Yat Sin david.yatsin@amd.com
Looks good to me. Please also post a link to the user mode change for this.
Note that the user mode code has not been merged upstream yet. I think this should be the final cleanup before the user mode CRIU plugin can be merged with the updated KFD version dependency.
Thanks, Felix
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 12 ++++++++---- include/uapi/linux/kfd_ioctl.h | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 2c7d76e67ddb..e1e2362841f8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1759,7 +1759,8 @@ static int criu_checkpoint_bos(struct kfd_process *p, goto exit; } }
if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
if (bo_bucket->alloc_flags& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) { ret = criu_get_prime_handle(&dumper_bo->tbo.base, bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,@@ -1812,7 +1813,8 @@ static int criu_checkpoint_bos(struct kfd_process *p,
exit: while (ret && bo_index--) {
if (bo_buckets[bo_index].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
if (bo_buckets[bo_index].alloc_flags& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) close_fd(bo_buckets[bo_index].dmabuf_fd);@@ -2211,7 +2213,8 @@ static int criu_restore_bo(struct kfd_process *p,
pr_debug("map memory was successful for the BO\n"); /* create the dmabuf object and export the bo */
- if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
- if (bo_bucket->alloc_flags
& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {@@ -2281,7 +2284,8 @@ static int criu_restore_bos(struct kfd_process *p,
exit: while (ret && i--) {
if (bo_buckets[i].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
if (bo_buckets[i].alloc_flags& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) close_fd(bo_buckets[i].dmabuf_fd);diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index b40687bf1014..eb9ff85f8556 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -33,9 +33,10 @@
- 1.5 - Add SVM API
- 1.6 - Query clear flags in SVM get_attr API
- 1.7 - Checkpoint Restore (CRIU) API
*/ #define KFD_IOCTL_MAJOR_VERSION 1
- 1.8 - CRIU - Support for SDMA transfers with GTT BOs
-#define KFD_IOCTL_MINOR_VERSION 7 +#define KFD_IOCTL_MINOR_VERSION 8
struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */
This is the link to the user mode change: https://github.com/checkpoint-restore/criu/pull/1709
Regards, David
-----Original Message----- From: Kuehling, Felix Felix.Kuehling@amd.com Sent: Tuesday, March 8, 2022 4:20 PM To: Yat Sin, David David.YatSin@amd.com; amd-gfx@lists.freedesktop.org; dri-devel@lists.freedesktop.org Subject: Re: [PATCH v2] drm/amdkfd: CRIU export dmabuf handles for GTT BOs
Am 2022-03-08 um 16:08 schrieb David Yat Sin:
Export dmabuf handles for GTT BOs so that their contents can be accessed using SDMA during checkpoint/restore.
Signed-off-by: David Yat Sin david.yatsin@amd.com
Looks good to me. Please also post a link to the user mode change for this.
Note that the user mode code has not been merged upstream yet. I think this should be the final cleanup before the user mode CRIU plugin can be merged with the updated KFD version dependency.
Thanks, Felix
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 12 ++++++++---- include/uapi/linux/kfd_ioctl.h | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 2c7d76e67ddb..e1e2362841f8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1759,7 +1759,8 @@ static int criu_checkpoint_bos(struct
kfd_process *p,
goto exit; } }
if (bo_bucket->alloc_flags &KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
if (bo_bucket->alloc_flags& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM |+KFD_IOC_ALLOC_MEM_FLAGS_GTT)) { ret = criu_get_prime_handle(&dumper_bo- tbo.base, bo_bucket->alloc_flags &
KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0, @@ - 1812,7
+1813,8 @@ static int criu_checkpoint_bos(struct kfd_process *p,
exit: while (ret && bo_index--) {
if (bo_buckets[bo_index].alloc_flags &KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
if (bo_buckets[bo_index].alloc_flags& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM |KFD_IOC_ALLOC_MEM_FLAGS_GTT))
close_fd(bo_buckets[bo_index].dmabuf_fd);}
@@ -2211,7 +2213,8 @@ static int criu_restore_bo(struct kfd_process *p,
pr_debug("map memory was successful for the BO\n"); /* create the dmabuf object and export the bo */
- if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
- if (bo_bucket->alloc_flags
& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM |KFD_IOC_ALLOC_MEM_FLAGS_GTT))
+{ ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base,
DRM_RDWR,
&bo_bucket->dmabuf_fd); if (ret)@@ -2281,7 +2284,8 @@ static int criu_restore_bos(struct kfd_process *p,
exit: while (ret && i--) {
if (bo_buckets[i].alloc_flags &KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
if (bo_buckets[i].alloc_flags& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM |KFD_IOC_ALLOC_MEM_FLAGS_GTT))
close_fd(bo_buckets[i].dmabuf_fd);} kvfree(bo_buckets); diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index b40687bf1014..eb9ff85f8556 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -33,9 +33,10 @@
- 1.5 - Add SVM API
- 1.6 - Query clear flags in SVM get_attr API
- 1.7 - Checkpoint Restore (CRIU) API
*/ #define KFD_IOCTL_MAJOR_VERSION 1
- 1.8 - CRIU - Support for SDMA transfers with GTT BOs
-#define KFD_IOCTL_MINOR_VERSION 7 +#define KFD_IOCTL_MINOR_VERSION 8
struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */
[AMD Official Use Only]
Please ignore the previous email, that was sent in error. This one is with the minor version bump so this looks good.
Reviewed-by : Rajneesh Bhardwaj rajneesh.bhardwaj@amd.com
-----Original Message----- From: amd-gfx amd-gfx-bounces@lists.freedesktop.org On Behalf Of David Yat Sin Sent: Tuesday, March 8, 2022 4:08 PM To: amd-gfx@lists.freedesktop.org; dri-devel@lists.freedesktop.org Cc: Kuehling, Felix Felix.Kuehling@amd.com; Yat Sin, David David.YatSin@amd.com Subject: [PATCH v2] drm/amdkfd: CRIU export dmabuf handles for GTT BOs
Export dmabuf handles for GTT BOs so that their contents can be accessed using SDMA during checkpoint/restore.
Signed-off-by: David Yat Sin david.yatsin@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 12 ++++++++---- include/uapi/linux/kfd_ioctl.h | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 2c7d76e67ddb..e1e2362841f8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1759,7 +1759,8 @@ static int criu_checkpoint_bos(struct kfd_process *p, goto exit; } } - if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { + if (bo_bucket->alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) +{ ret = criu_get_prime_handle(&dumper_bo->tbo.base, bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0, @@ -1812,7 +1813,8 @@ static int criu_checkpoint_bos(struct kfd_process *p,
exit: while (ret && bo_index--) { - if (bo_buckets[bo_index].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) + if (bo_buckets[bo_index].alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) close_fd(bo_buckets[bo_index].dmabuf_fd); }
@@ -2211,7 +2213,8 @@ static int criu_restore_bo(struct kfd_process *p,
pr_debug("map memory was successful for the BO\n"); /* create the dmabuf object and export the bo */ - if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { + if (bo_bucket->alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) { ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR, &bo_bucket->dmabuf_fd); if (ret) @@ -2281,7 +2284,8 @@ static int criu_restore_bos(struct kfd_process *p,
exit: while (ret && i--) { - if (bo_buckets[i].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) + if (bo_buckets[i].alloc_flags + & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) close_fd(bo_buckets[i].dmabuf_fd); } kvfree(bo_buckets); diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index b40687bf1014..eb9ff85f8556 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -33,9 +33,10 @@ * - 1.5 - Add SVM API * - 1.6 - Query clear flags in SVM get_attr API * - 1.7 - Checkpoint Restore (CRIU) API + * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 7 +#define KFD_IOCTL_MINOR_VERSION 8
struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ -- 2.17.1
dri-devel@lists.freedesktop.org
-
Bhardwaj, Rajneesh -
David Yat Sin -
Felix Kuehling -
Yat Sin, David