From: Thierry Reding treding@nvidia.com
The GPU integrated in NVIDIA Tegra SoCs is connected to system memory via two paths: one direct path to the memory controller and another path that goes through a system MMU first. It's not typically necessary to go through the system MMU because the GPU's MMU can already map buffers so that they appear contiguous to the GPU.
However, in order to support big pages, the system MMU has to be used to combine multiple small pages into one virtually contiguous chunk so that the GPU can then treat that as a single big page.
In order to prepare for big page support, implement a custom MMU class that takes care of setting the IOMMU bit when writing page tables and when appropriate.
This is also necessary to make sure that Nouveau works correctly on Tegra devices where the GPU is connected to a system MMU and that IOMMU is used to back the DMA API. Currently Nouveau assumes that the DMA API is never backed by an IOMMU, so access to DMA-mapped buffers fault when suddenly this assumption is no longer true.
One situation where this can happen is on 32-bit Tegra SoCs where the ARM architecture code automatically attaches the GPU with a DMA/IOMMU domain. This is currently worked around by detaching the GPU from the IOMMU domain at probe time. However, with Tegra186 and later this can now also happen, but unfortunately no mechanism exists to detach from the domain in the 64-bit ARM architecture code.
Using this Tegra-specific MMU class ensures that DMA-mapped buffers are properly mapped (with the IOMMU bit set) if the DMA API is backed by an IOMMU domain.
Signed-off-by: Thierry Reding treding@nvidia.com --- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 50 ++++++++++++++++++- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.h | 44 ++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 6 ++- .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 4 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 1 + .../drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c | 22 +++++++- .../drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c | 4 +- .../drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c | 20 +++++++- 8 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.h
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index ac74965a60d4..d9a5e05b7dc7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -19,11 +19,59 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ + +#include "gk20a.h" #include "mem.h" #include "vmm.h"
+#include <nvkm/core/tegra.h> #include <nvif/class.h>
+static void +gk20a_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device, + int index, struct gk20a_mmu *mmu) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(device->dev); + struct nvkm_device_tegra *tegra = device->func->tegra(device); + + nvkm_mmu_ctor(func, device, index, &mmu->base); + + /* + * If the DMA API is backed by an IOMMU, make sure the IOMMU bit is + * set for all buffer accesses. If the IOMMU is explicitly used, it + * is only used for instance blocks and the MMU doesn't care, since + * buffer objects are only mapped through the MMU, not through the + * IOMMU. + * + * Big page support could be implemented using explicit IOMMU usage, + * but the DMA API already provides that for free, so we don't worry + * about it for now. + */ + if (domain && !tegra->iommu.domain) { + mmu->iommu_mask = BIT_ULL(tegra->func->iommu_bit); + nvkm_debug(&mmu->base.subdev, "IOMMU mask: %llx\n", + mmu->iommu_mask); + } +} + +int +gk20a_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device, + int index, struct nvkm_mmu **pmmu) +{ + struct gk20a_mmu *mmu; + + mmu = kzalloc(sizeof(*mmu), GFP_KERNEL); + if (!mmu) + return -ENOMEM; + + gk20a_mmu_ctor(func, device, index, mmu); + + if (pmmu) + *pmmu = &mmu->base; + + return 0; +} + static const struct nvkm_mmu_func gk20a_mmu = { .dma_bits = 40, @@ -37,5 +85,5 @@ gk20a_mmu = { int gk20a_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { - return nvkm_mmu_new_(&gk20a_mmu, device, index, pmmu); + return gk20a_mmu_new_(&gk20a_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.h new file mode 100644 index 000000000000..bb81fc62509c --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 NVIDIA Corporation. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __NVKM_MMU_GK20A_H__ +#define __NVKM_MMU_GK20A_H__ + +#include "priv.h" + +struct gk20a_mmu { + struct nvkm_mmu base; + + /* + * If an IOMMU is used, indicates which address bit will trigger an + * IOMMU translation when set (when this bit is not set, the IOMMU is + * bypassed). A value of 0 means an IOMMU is never used. + */ + u64 iommu_mask; +}; + +#define gk20a_mmu(mmu) container_of(mmu, struct gk20a_mmu, base) + +int gk20a_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *, + int index, struct nvkm_mmu **); + +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 7353a94b4091..7fccd4df52a8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -19,6 +19,8 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ + +#include "gk20a.h" #include "mem.h" #include "vmm.h"
@@ -50,6 +52,6 @@ int gm20b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { if (device->fb->page) - return nvkm_mmu_new_(&gm20b_mmu_fixed, device, index, pmmu); - return nvkm_mmu_new_(&gm20b_mmu, device, index, pmmu); + return gk20a_mmu_new_(&gm20b_mmu_fixed, device, index, pmmu); + return gk20a_mmu_new_(&gm20b_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index 0a50be9a785a..ae3cb47be3d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -19,6 +19,8 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ + +#include "gk20a.h" #include "mem.h" #include "vmm.h"
@@ -41,5 +43,5 @@ gp10b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", true)) return gm20b_mmu_new(device, index, pmmu); - return nvkm_mmu_new_(&gp10b_mmu, device, index, pmmu); + return gk20a_mmu_new_(&gp10b_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 5e55ecbd8005..fb3a9e8bb9cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -213,6 +213,7 @@ void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type); void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr);
int gk20a_vmm_aper(enum nvkm_memory_target); +int gk20a_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, struct nvkm_mmu *, bool, u64, u64, void *, u32, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c index 5a9582dce970..16d7bf727292 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c @@ -19,6 +19,8 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ + +#include "gk20a.h" #include "vmm.h"
#include <core/memory.h> @@ -33,12 +35,28 @@ gk20a_vmm_aper(enum nvkm_memory_target target) } }
+int +gk20a_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, + struct nvkm_vmm_map *map) +{ + struct gk20a_mmu *mmu = gk20a_mmu(vmm->mmu); + int ret; + + ret = gf100_vmm_valid(vmm, argv, argc, map); + if (ret < 0) + return ret; + + map->type |= mmu->iommu_mask >> 8; + + return 0; +} + static const struct nvkm_vmm_func gk20a_vmm_17 = { .join = gf100_vmm_join, .part = gf100_vmm_part, .aper = gf100_vmm_aper, - .valid = gf100_vmm_valid, + .valid = gk20a_vmm_valid, .flush = gf100_vmm_flush, .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { @@ -53,7 +71,7 @@ gk20a_vmm_16 = { .join = gf100_vmm_join, .part = gf100_vmm_part, .aper = gf100_vmm_aper, - .valid = gf100_vmm_valid, + .valid = gk20a_vmm_valid, .flush = gf100_vmm_flush, .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c index 96b759695dd8..7a6066d886cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c @@ -26,7 +26,7 @@ gm20b_vmm_17 = { .join = gm200_vmm_join, .part = gf100_vmm_part, .aper = gk20a_vmm_aper, - .valid = gf100_vmm_valid, + .valid = gk20a_vmm_valid, .flush = gf100_vmm_flush, .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { @@ -42,7 +42,7 @@ gm20b_vmm_16 = { .join = gm200_vmm_join, .part = gf100_vmm_part, .aper = gk20a_vmm_aper, - .valid = gf100_vmm_valid, + .valid = gk20a_vmm_valid, .flush = gf100_vmm_flush, .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c index e081239afe58..180c8f006e32 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c @@ -19,14 +19,32 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ + +#include "gk20a.h" #include "vmm.h"
+static int +gp10b_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, + struct nvkm_vmm_map *map) +{ + struct gk20a_mmu *mmu = gk20a_mmu(vmm->mmu); + int ret; + + ret = gp100_vmm_valid(vmm, argv, argc, map); + if (ret < 0) + return ret; + + map->type |= mmu->iommu_mask >> 4; + + return 0; +} + static const struct nvkm_vmm_func gp10b_vmm = { .join = gp100_vmm_join, .part = gf100_vmm_part, .aper = gk20a_vmm_aper, - .valid = gp100_vmm_valid, + .valid = gp10b_vmm_valid, .flush = gp100_vmm_flush, .mthd = gp100_vmm_mthd, .invalidate_pdb = gp100_vmm_invalidate_pdb,