As single buffer object may be mapped into different address spaces at the same time. For now we only have two different address spaces for the 3D and 2D pipe, but this may change as soon as we implement per-process page tables.
Allow this by having each buffer object manage a list of all it's mappings into the respective address spaces.
Signed-off-by: Lucas Stach l.stach@pengutronix.de --- drivers/staging/etnaviv/etnaviv_gem.c | 44 +++++++++++++++++----- drivers/staging/etnaviv/etnaviv_gem.h | 13 ++++++- drivers/staging/etnaviv/etnaviv_mmu.c | 71 +++++++++++++++++++++-------------- drivers/staging/etnaviv/etnaviv_mmu.h | 8 +++- 4 files changed, 93 insertions(+), 43 deletions(-)
diff --git a/drivers/staging/etnaviv/etnaviv_gem.c b/drivers/staging/etnaviv/etnaviv_gem.c index 57f3080fb632..04594dad27e2 100644 --- a/drivers/staging/etnaviv/etnaviv_gem.c +++ b/drivers/staging/etnaviv/etnaviv_gem.c @@ -310,19 +310,25 @@ int etnaviv_gem_get_iova_locked(struct etnaviv_gpu *gpu, struct drm_gem_object *obj, uint32_t *iova) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); + struct etnaviv_vram_mapping *mapping = + etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu); int ret = 0;
- if (!etnaviv_obj->iova && !(etnaviv_obj->flags & ETNA_BO_CMDSTREAM)) { + if (etnaviv_obj->flags & ETNA_BO_CMDSTREAM) { + *iova = etnaviv_obj->paddr; + return 0; + } + + if (!mapping) { struct page **pages = etnaviv_gem_get_pages(etnaviv_obj); if (IS_ERR(pages)) return PTR_ERR(pages); - ret = etnaviv_iommu_map_gem(gpu->mmu, etnaviv_obj, - gpu->memory_base); + gpu->memory_base, &mapping); }
if (!ret) - *iova = etnaviv_obj->iova; + *iova = mapping->iova;
return ret; } @@ -331,13 +337,15 @@ int etnaviv_gem_get_iova(struct etnaviv_gpu *gpu, struct drm_gem_object *obj, int id, uint32_t *iova) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); + struct etnaviv_vram_mapping *mapping = + etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu); int ret;
/* this is safe right now because we don't unmap until the * bo is deleted: */ - if (etnaviv_obj->iova) { - *iova = etnaviv_obj->iova; + if (mapping) { + *iova = mapping->iova; return 0; }
@@ -546,11 +554,12 @@ static const struct etnaviv_gem_ops etnaviv_gem_cmd_ops = { static void etnaviv_free_obj(struct drm_gem_object *obj) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); - struct etnaviv_drm_private *priv = obj->dev->dev_private; - struct etnaviv_iommu *mmu = priv->mmu; + struct etnaviv_vram_mapping *mapping, *tmp;
- if (mmu) - etnaviv_iommu_unmap_gem(mmu, etnaviv_obj); + list_for_each_entry_safe(mapping, tmp, &etnaviv_obj->vram_list, + obj_head) { + etnaviv_iommu_unmap_gem(mapping->mmu, etnaviv_obj, mapping); + } }
static void etnaviv_gem_shmem_release(struct etnaviv_gem_object *etnaviv_obj) @@ -665,6 +674,7 @@ static int etnaviv_gem_new_impl(struct drm_device *dev, reservation_object_init(&etnaviv_obj->_resv);
INIT_LIST_HEAD(&etnaviv_obj->submit_entry); + INIT_LIST_HEAD(&etnaviv_obj->vram_list); list_add_tail(&etnaviv_obj->mm_list, &priv->inactive_list);
*obj = &etnaviv_obj->base; @@ -724,6 +734,20 @@ int etnaviv_gem_new_private(struct drm_device *dev, size_t size, uint32_t flags, return 0; }
+struct etnaviv_vram_mapping * +etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj, + struct etnaviv_iommu *mmu) +{ + struct etnaviv_vram_mapping *mapping; + + list_for_each_entry(mapping, &obj->vram_list, obj_head) { + if (mapping->mmu == mmu) + return mapping; + } + + return NULL; +} + struct get_pages_work { struct work_struct work; struct mm_struct *mm; diff --git a/drivers/staging/etnaviv/etnaviv_gem.h b/drivers/staging/etnaviv/etnaviv_gem.h index fadd5198b3e8..b0e5e968912a 100644 --- a/drivers/staging/etnaviv/etnaviv_gem.h +++ b/drivers/staging/etnaviv/etnaviv_gem.h @@ -30,6 +30,13 @@ struct etnaviv_gem_userptr { bool ro; };
+struct etnaviv_vram_mapping { + struct list_head obj_head; + struct etnaviv_iommu *mmu; + struct drm_mm_node vram_node; + uint32_t iova; +}; + struct etnaviv_gem_object { struct drm_gem_object base; const struct etnaviv_gem_ops *ops; @@ -59,7 +66,6 @@ struct etnaviv_gem_object { struct page **pages; struct sg_table *sgt; void *vaddr; - uint32_t iova;
/* for ETNA_BO_CMDSTREAM */ dma_addr_t paddr; @@ -68,7 +74,7 @@ struct etnaviv_gem_object { struct reservation_object *resv; struct reservation_object _resv;
- struct drm_mm_node *gpu_vram_node; + struct list_head vram_list;
/* for buffer manipulation during submit */ u32 offset; @@ -120,6 +126,9 @@ struct etnaviv_gem_submit { } bos[0]; };
+struct etnaviv_vram_mapping * +etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj, + struct etnaviv_iommu *mmu); int etnaviv_gem_new_private(struct drm_device *dev, size_t size, uint32_t flags, struct etnaviv_gem_object **res); struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *obj); diff --git a/drivers/staging/etnaviv/etnaviv_mmu.c b/drivers/staging/etnaviv/etnaviv_mmu.c index 897356d08e30..4bcc3eabb3a1 100644 --- a/drivers/staging/etnaviv/etnaviv_mmu.c +++ b/drivers/staging/etnaviv/etnaviv_mmu.c @@ -92,28 +92,38 @@ int etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, uint32_t iova, }
int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, - struct etnaviv_gem_object *etnaviv_obj, uint32_t memory_base) + struct etnaviv_gem_object *etnaviv_obj, uint32_t memory_base, + struct etnaviv_vram_mapping **out_mapping) { struct etnaviv_drm_private *priv = etnaviv_obj->base.dev->dev_private; struct sg_table *sgt = etnaviv_obj->sgt; + struct etnaviv_vram_mapping *mapping, *free = NULL; struct drm_mm_node *node; int ret;
+ mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + + INIT_LIST_HEAD(&mapping->obj_head); + mapping->mmu = mmu; + /* v1 MMU can optimize single entry (contiguous) scatterlists */ if (sgt->nents == 1) { uint32_t iova;
iova = sg_dma_address(sgt->sgl) - memory_base; if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) { - etnaviv_obj->iova = iova; + mapping->iova = iova; + list_add_tail(&mapping->obj_head, + &etnaviv_obj->vram_list); + if (out_mapping) + *out_mapping = mapping; return 0; } }
- node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return -ENOMEM; - + node = &mapping->vram_node; while (1) { struct etnaviv_gem_object *o, *n; struct list_head list; @@ -142,8 +152,8 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, found = 0; INIT_LIST_HEAD(&list); list_for_each_entry(o, &priv->inactive_list, mm_list) { - if (!o->gpu_vram_node || - o->gpu_vram_node->mm != &mmu->mm) + free = etnaviv_gem_get_vram_mapping(o, mmu); + if (!free) continue;
/* @@ -154,7 +164,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, continue;
list_add(&o->submit_entry, &list); - if (drm_mm_scan_add_block(o->gpu_vram_node)) { + if (drm_mm_scan_add_block(&free->vram_node)) { found = true; break; } @@ -163,7 +173,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, if (!found) { /* Nothing found, clean up and fail */ list_for_each_entry_safe(o, n, &list, submit_entry) - BUG_ON(drm_mm_scan_remove_block(o->gpu_vram_node)); + BUG_ON(drm_mm_scan_remove_block(&free->vram_node)); break; }
@@ -174,12 +184,12 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, * can leave the block pinned. */ list_for_each_entry_safe(o, n, &list, submit_entry) - if (!drm_mm_scan_remove_block(o->gpu_vram_node)) + if (!drm_mm_scan_remove_block(&free->vram_node)) list_del_init(&o->submit_entry);
list_for_each_entry_safe(o, n, &list, submit_entry) { list_del_init(&o->submit_entry); - etnaviv_iommu_unmap_gem(mmu, o); + etnaviv_iommu_unmap_gem(mmu, o, free); }
/* @@ -191,40 +201,43 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, }
if (ret < 0) { - kfree(node); + kfree(mapping); return ret; }
mmu->last_iova = node->start + etnaviv_obj->base.size; - etnaviv_obj->iova = node->start; - etnaviv_obj->gpu_vram_node = node; + mapping->iova = node->start; ret = etnaviv_iommu_map(mmu, node->start, sgt, etnaviv_obj->base.size, IOMMU_READ | IOMMU_WRITE);
if (ret < 0) { drm_mm_remove_node(node); - kfree(node); - - etnaviv_obj->iova = 0; - etnaviv_obj->gpu_vram_node = NULL; + kfree(mapping); + return ret; }
+ list_add_tail(&mapping->obj_head, &etnaviv_obj->vram_list); + if (out_mapping) + *out_mapping = mapping; + return ret; }
void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu, - struct etnaviv_gem_object *etnaviv_obj) + struct etnaviv_gem_object *etnaviv_obj, + struct etnaviv_vram_mapping *mapping) { - if (etnaviv_obj->gpu_vram_node) { - uint32_t offset = etnaviv_obj->gpu_vram_node->start; + if (mapping) { + uint32_t offset = mapping->vram_node.start;
- etnaviv_iommu_unmap(mmu, offset, etnaviv_obj->sgt, - etnaviv_obj->base.size); - drm_mm_remove_node(etnaviv_obj->gpu_vram_node); - kfree(etnaviv_obj->gpu_vram_node); - - etnaviv_obj->gpu_vram_node = NULL; - etnaviv_obj->iova = 0; + if (mapping->iova >= 0x80000000) { + etnaviv_iommu_unmap(mmu, offset, etnaviv_obj->sgt, + etnaviv_obj->base.size); + drm_mm_remove_node(&mapping->vram_node); + } + list_del(&mapping->obj_head); + kfree(mapping); + mapping = NULL; } }
diff --git a/drivers/staging/etnaviv/etnaviv_mmu.h b/drivers/staging/etnaviv/etnaviv_mmu.h index 29881e27dc7e..9a4b493015f6 100644 --- a/drivers/staging/etnaviv/etnaviv_mmu.h +++ b/drivers/staging/etnaviv/etnaviv_mmu.h @@ -20,6 +20,8 @@
#include <linux/iommu.h>
+struct etnaviv_vram_mapping; + struct etnaviv_iommu { struct drm_device *dev; struct iommu_domain *domain; @@ -39,9 +41,11 @@ int etnaviv_iommu_map(struct etnaviv_iommu *iommu, uint32_t iova, int etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, uint32_t iova, struct sg_table *sgt, unsigned len); int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, - struct etnaviv_gem_object *etnaviv_obj, uint32_t memory_base); + struct etnaviv_gem_object *etnaviv_obj, uint32_t memory_base, + struct etnaviv_vram_mapping **mapping); void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu, - struct etnaviv_gem_object *etnaviv_obj); + struct etnaviv_gem_object *etnaviv_obj, + struct etnaviv_vram_mapping *mapping); void etnaviv_iommu_destroy(struct etnaviv_iommu *iommu);
struct etnaviv_iommu *etnaviv_iommu_new(struct drm_device *dev,