On 25/01/2021 21:57, Jianxin Xiong wrote:
Define a new sub-class of 'MR' that uses dma-buf object for the memory region. Define a new class 'DmaBuf' as a wrapper for dma-buf allocation mechanism implemented in C.
Update the cmake function for cython modules to allow building modules with mixed cython and c source files.
Signed-off-by: Jianxin Xiong jianxin.xiong@intel.com
buildlib/pyverbs_functions.cmake | 78 +++++++---- pyverbs/CMakeLists.txt | 11 +- pyverbs/dmabuf.pxd | 15 +++ pyverbs/dmabuf.pyx | 73 ++++++++++ pyverbs/dmabuf_alloc.c | 278 +++++++++++++++++++++++++++++++++++++++ pyverbs/dmabuf_alloc.h | 19 +++ pyverbs/libibverbs.pxd | 2 + pyverbs/mr.pxd | 6 + pyverbs/mr.pyx | 105 ++++++++++++++- 9 files changed, 557 insertions(+), 30 deletions(-) create mode 100644 pyverbs/dmabuf.pxd create mode 100644 pyverbs/dmabuf.pyx create mode 100644 pyverbs/dmabuf_alloc.c create mode 100644 pyverbs/dmabuf_alloc.h
diff --git a/buildlib/pyverbs_functions.cmake b/buildlib/pyverbs_functions.cmake index 953cec2..0792410 100644 --- a/buildlib/pyverbs_functions.cmake +++ b/buildlib/pyverbs_functions.cmake @@ -1,35 +1,61 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file +# Copyright (c) 2020, Intel Corporation. All rights reserved. See COPYING file
+function(build_module_from_cfiles PY_MODULE MODULE_NAME ALL_CFILES LINKER_FLAGS)
- string(REGEX REPLACE "\.so$" "" SONAME "${MODULE_NAME}${CMAKE_PYTHON_SO_SUFFIX}")
- add_library(${SONAME} SHARED ${ALL_CFILES})
- set_target_properties(${SONAME} PROPERTIES
- COMPILE_FLAGS "${CMAKE_C_FLAGS} -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option -Wno-deprecated-declarations ${NO_VAR_TRACKING_FLAGS}"
- LIBRARY_OUTPUT_DIRECTORY "${BUILD_PYTHON}/${PY_MODULE}"
- PREFIX "")
- target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs rdmacm ${LINKER_FLAGS})
- install(TARGETS ${SONAME}
- DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE})
+endfunction()
function(rdma_cython_module PY_MODULE LINKER_FLAGS)
- foreach(PYX_FILE ${ARGN})
- get_filename_component(FILENAME ${PYX_FILE} NAME_WE)
- get_filename_component(DIR ${PYX_FILE} DIRECTORY)
- if (DIR)
set(PYX "${CMAKE_CURRENT_SOURCE_DIR}/${DIR}/${FILENAME}.pyx")- else()
set(PYX "${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}.pyx")- endif()
- set(CFILE "${CMAKE_CURRENT_BINARY_DIR}/${FILENAME}.c")
- include_directories(${PYTHON_INCLUDE_DIRS})
- add_custom_command(
OUTPUT "${CFILE}"MAIN_DEPENDENCY "${PYX}"COMMAND ${CYTHON_EXECUTABLE} "${PYX}" -o "${CFILE}""-I${PYTHON_INCLUDE_DIRS}"COMMENT "Cythonizing ${PYX}"
- set(ALL_CFILES "")
- set(MODULE_NAME "")
- foreach(SRC_FILE ${ARGN})
- get_filename_component(FILENAME ${SRC_FILE} NAME_WE)
- get_filename_component(DIR ${SRC_FILE} DIRECTORY)
- get_filename_component(EXT ${SRC_FILE} EXT)
- if (DIR)
set(SRC_PATH "${CMAKE_CURRENT_SOURCE_DIR}/${DIR}")- else()
set(SRC_PATH "${CMAKE_CURRENT_SOURCE_DIR}")- endif()
- if (${EXT} STREQUAL ".pyx")
# each .pyx file starts a new module, finish the previous module firstif (ALL_CFILES AND MODULE_NAME)build_module_from_cfiles(${PY_MODULE} ${MODULE_NAME} "${ALL_CFILES}" "${LINKER_FLAGS}")endif()set(PYX "${SRC_PATH}/${FILENAME}.pyx")set(CFILE "${CMAKE_CURRENT_BINARY_DIR}/${FILENAME}.c")include_directories(${PYTHON_INCLUDE_DIRS})add_custom_command(OUTPUT "${CFILE}"MAIN_DEPENDENCY "${PYX}"COMMAND ${CYTHON_EXECUTABLE} "${PYX}" -o "${CFILE}""-I${PYTHON_INCLUDE_DIRS}"COMMENT "Cythonizing ${PYX}" )
- string(REGEX REPLACE "\.so$" "" SONAME "${FILENAME}${CMAKE_PYTHON_SO_SUFFIX}")
- add_library(${SONAME} SHARED ${CFILE})
- set_target_properties(${SONAME} PROPERTIES
COMPILE_FLAGS "${CMAKE_C_FLAGS} -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option -Wno-deprecated-declarations ${NO_VAR_TRACKING_FLAGS}"LIBRARY_OUTPUT_DIRECTORY "${BUILD_PYTHON}/${PY_MODULE}"PREFIX "")- target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs rdmacm ${LINKER_FLAGS})
- install(TARGETS ${SONAME}
DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE})
set(MODULE_NAME ${FILENAME})set(ALL_CFILES "${CFILE}")- elseif(${EXT} STREQUAL ".c")
# .c files belong to the same module as the most recent .pyx file,# ignored if appearing before all .pyx filesset(CFILE "${SRC_PATH}/${FILENAME}.c")set(ALL_CFILES "${ALL_CFILES};${CFILE}")- else()
continue()- endif() endforeach()
- # finish the last module
- if (ALL_CFILES AND MODULE_NAME)
- build_module_from_cfiles(${PY_MODULE} ${MODULE_NAME} "${ALL_CFILES}" "${LINKER_FLAGS}")
- endif()
endfunction()
function(rdma_python_module PY_MODULE) diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 9542c4b..6fd7625 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -1,5 +1,10 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file +# Copyright (c) 2020, Intel Corporation. All rights reserved. See COPYING file
+publish_internal_headers(""
- dmabuf_alloc.h
+)
rdma_cython_module(pyverbs "" addr.pyx @@ -8,15 +13,17 @@ rdma_cython_module(pyverbs "" cmid.pyx cq.pyx device.pyx
- dmabuf.pyx
- dmabuf_alloc.c enums.pyx mem_alloc.pyx mr.pyx pd.pyx qp.pyx
- srq.pyx wr.pyx xrcd.pyx
- srq.pyx
- )
+)
rdma_python_module(pyverbs __init__.py diff --git a/pyverbs/dmabuf.pxd b/pyverbs/dmabuf.pxd new file mode 100644 index 0000000..a063acb --- /dev/null +++ b/pyverbs/dmabuf.pxd @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2020, Intel Corporation. All rights reserved. See COPYING file
+#cython: language_level=3
+cdef class DmaBuf:
- cdef int drm_fd
- cdef int handle
- cdef int fd
- cdef unsigned long size
- cdef unsigned long map_offset
- cdef void *dmabuf
- cdef object dmabuf_mrs
- cdef add_ref(self, obj)
- cpdef close(self)
diff --git a/pyverbs/dmabuf.pyx b/pyverbs/dmabuf.pyx new file mode 100644 index 0000000..b9406bd --- /dev/null +++ b/pyverbs/dmabuf.pyx @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2020, Intel Corporation. All rights reserved. See COPYING file
+#cython: language_level=3
+import weakref
+from pyverbs.base cimport close_weakrefs +from pyverbs.base import PyverbsRDMAErrno +from pyverbs.mr cimport DmaBufMR
+cdef extern from "dmabuf_alloc.h":
- cdef struct dmabuf:
pass- dmabuf *dmabuf_alloc(unsigned long size, int unit, int gtt)
- void dmabuf_free(dmabuf *dmabuf)
- int dmabuf_get_drm_fd(dmabuf *dmabuf)
- int dmabuf_get_fd(dmabuf *dmabuf)
- unsigned long dmabuf_get_offset(dmabuf *dmabuf)
+cdef class DmaBuf:
- def __init__(self, size, unit=0, gtt=0):
"""Allocate DmaBuf object from a GPU device. This is done through theDRI device interface. Usually this requires the effective user idbeing a member of the 'render' group.:param size: The size (in number of bytes) of the buffer.:param unit: The unit number of the GPU to allocate the buffer from.:param gtt: Allocate from GTT instead of VRAM.:return: The newly created DmaBuf object on success."""self.dmabuf_mrs = weakref.WeakSet()self.dmabuf = dmabuf_alloc(size, unit, gtt)if self.dmabuf == NULL:raise PyverbsRDMAErrno(f'Failed to allocate dmabuf of size {size} on unit {unit}')self.drm_fd = dmabuf_get_drm_fd(<dmabuf *>self.dmabuf)self.fd = dmabuf_get_fd(<dmabuf *>self.dmabuf)self.map_offset = dmabuf_get_offset(<dmabuf *>self.dmabuf)- def __dealloc__(self):
self.close()- cpdef close(self):
if self.dmabuf == NULL:return Noneclose_weakrefs([self.dmabuf_mrs])dmabuf_free(<dmabuf *>self.dmabuf)self.dmabuf = NULL- cdef add_ref(self, obj):
if isinstance(obj, DmaBufMR):self.dmabuf_mrs.add(obj)- @property
- def drm_fd(self):
return self.drm_fd- @property
- def handle(self):
return self.handle- @property
- def fd(self):
return self.fd- @property
- def size(self):
return self.size- @property
- def map_offset(self):
return self.map_offsetdiff --git a/pyverbs/dmabuf_alloc.c b/pyverbs/dmabuf_alloc.c new file mode 100644 index 0000000..05eae75 --- /dev/null +++ b/pyverbs/dmabuf_alloc.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/*
- Copyright 2020 Intel Corporation. All rights reserved. See COPYING file
- */
+#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> +#include <drm/drm.h> +#include <drm/i915_drm.h> +#include <drm/amdgpu_drm.h> +#include <drm/radeon_drm.h>
I assume these should come from the kernel headers package, right? We're getting new compilation errors on SLES 15 for some reason, although the package is installed.
FAILED: pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o /usr/bin/cc -Ddmabuf_cpython_36m_x86_64_linux_gnu_EXPORTS -Iinclude -I/usr/include/libnl3 -I/usr/include/python3.6m -Wall -Wextra -Wno-sign-compare -Wno-unused-parameter -Wmissing-prototypes -Wmissing-declarations -Wwrite-strings -Wformat=2 -Wformat-nonliteral -Wdate-time -Wnested-externs -Wshadow -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -O2 -g -fPIC -Wall -Wextra -Wno-sign-compare -Wno-unused-parameter -Wmissing-prototypes -Wmissing-declarations -Wwrite-strings -Wformat=2 -Wformat-nonliteral -Wdate-time -Wnested-externs -Wshadow -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option -Wno-deprecated-declarations -fno-var-tracking-assignments -std=gnu11 -MD -MT pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o -MF pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o.d -o pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o -c ../pyverbs/dmabuf_alloc.c ../pyverbs/dmabuf_alloc.c:12:10: fatal error: drm/drm.h: No such file or directory #include <drm/drm.h> ^~~~~~~~~~~ compilation terminated.