drm/nouveau/imem/nv50: embed nvkm_instobj directly into nv04_instobj

This is not as simple as it was for earlier GPUs, due to the need to swap
accessor functions depending on whether BAR2 is usable or not.

We were previously protected by nvkm_instobj's accessor functions keeping
an object mapped permanently, with some unclear magic that managed to hit
the slow-path where needed even if an object was marked as mapped.

That's been replaced here by reference counting maps (some objects, like
page tables can be accessed concurrently), and swapping the functions as
necessary.

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

Author: Ben Skeggs

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c

@@ -37,13 +37,14 @@ struct nv50_instmem {
 /******************************************************************************
  * instmem object implementation
  *****************************************************************************/
-#define nv50_instobj(p) container_of((p), struct nv50_instobj, memory)
+#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
 
 struct nv50_instobj {
-	struct nvkm_memory memory;
+	struct nvkm_instobj base;
 	struct nv50_instmem *imem;
 	struct nvkm_mem *mem;
 	struct nvkm_vma bar;
+	refcount_t maps;
 	void *map;
 };
 
@@ -93,31 +94,59 @@ nv50_instobj_slow = {
 	.wr32 = nv50_instobj_wr32_slow,
 };
 
+static void
+nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
+{
+	iowrite32_native(data, nv50_instobj(memory)->map + offset);
+}
+
+static u32
+nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
+{
+	return ioread32_native(nv50_instobj(memory)->map + offset);
+}
+
+static const struct nvkm_memory_ptrs
+nv50_instobj_fast = {
+	.rd32 = nv50_instobj_rd32,
+	.wr32 = nv50_instobj_wr32,
+};
+
 static void
 nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
 {
-	struct nvkm_memory *memory = &iobj->memory;
-	struct nvkm_subdev *subdev = &iobj->imem->base.subdev;
+	struct nv50_instmem *imem = iobj->imem;
+	struct nvkm_memory *memory = &iobj->base.memory;
+	struct nvkm_subdev *subdev = &imem->base.subdev;
 	struct nvkm_device *device = subdev->device;
+	struct nvkm_vma bar = {};
 	u64 size = nvkm_memory_size(memory);
-	void __iomem *map;
 	int ret;
 
-	iobj->map = ERR_PTR(-ENOMEM);
-
-	ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar);
-	if (ret == 0) {
-		map = ioremap(device->func->resource_addr(device, 3) +
-			      (u32)iobj->bar.offset, size);
-		if (map) {
-			nvkm_memory_map(memory, &iobj->bar, 0);
-			iobj->map = map;
-		} else {
-			nvkm_warn(subdev, "PRAMIN ioremap failed\n");
-			nvkm_vm_put(&iobj->bar);
-		}
-	} else {
-		nvkm_warn(subdev, "PRAMIN exhausted\n");
+	/* Attempt to allocate BAR2 address-space and map the object
+	 * into it.  The lock has to be dropped while doing this due
+	 * to the possibility of recursion for page table allocation.
+	 */
+	mutex_unlock(&subdev->mutex);
+	ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar);
+	if (ret == 0)
+		nvkm_memory_map(memory, &bar, 0);
+	mutex_lock(&subdev->mutex);
+	if (ret || iobj->bar.node) {
+		/* We either failed, or another thread beat us. */
+		mutex_unlock(&subdev->mutex);
+		nvkm_vm_put(&bar);
+		mutex_lock(&subdev->mutex);
+		return;
+	}
+
+	/* Make the mapping visible to the host. */
+	iobj->bar = bar;
+	iobj->map = ioremap(device->func->resource_addr(device, 3) +
+			    (u32)iobj->bar.offset, size);
+	if (!iobj->map) {
+		nvkm_warn(subdev, "PRAMIN ioremap failed\n");
+		nvkm_vm_put(&iobj->bar);
 	}
 }
 
@@ -131,28 +160,66 @@ nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset)
 static void
 nv50_instobj_release(struct nvkm_memory *memory)
 {
+	struct nv50_instobj *iobj = nv50_instobj(memory);
+	struct nv50_instmem *imem = iobj->imem;
+	struct nvkm_subdev *subdev = &imem->base.subdev;
+
+	nvkm_bar_flush(subdev->device->bar);
+
+	if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) {
+		/* Switch back to NULL accessors when last map is gone. */
+		iobj->base.memory.ptrs = &nv50_instobj_slow;
+		mutex_unlock(&subdev->mutex);
+	}
 }
 
 static void __iomem *
 nv50_instobj_acquire(struct nvkm_memory *memory)
 {
 	struct nv50_instobj *iobj = nv50_instobj(memory);
-	struct nv50_instmem *imem = iobj->imem;
-	struct nvkm_vm *vm;
+	struct nvkm_instmem *imem = &iobj->imem->base;
+	struct nvkm_vmm *vmm;
+	void __iomem *map = NULL;
 
-	if (!iobj->map && (vm = nvkm_bar_bar2_vmm(imem->base.subdev.device)))
-		nv50_instobj_kmap(iobj, vm);
-	if (!IS_ERR_OR_NULL(iobj->map))
+	/* Already mapped? */
+	if (refcount_inc_not_zero(&iobj->maps))
 		return iobj->map;
 
-	return NULL;
+	/* Take the lock, and re-check that another thread hasn't
+	 * already mapped the object in the meantime.
+	 */
+	mutex_lock(&imem->subdev.mutex);
+	if (refcount_inc_not_zero(&iobj->maps)) {
+		mutex_unlock(&imem->subdev.mutex);
+		return iobj->map;
+	}
+
+	/* Attempt to get a direct CPU mapping of the object. */
+	if (!iobj->map && (vmm = nvkm_bar_bar2_vmm(imem->subdev.device)))
+		nv50_instobj_kmap(iobj, vmm);
+	map = iobj->map;
+
+	if (!refcount_inc_not_zero(&iobj->maps)) {
+		if (map)
+			iobj->base.memory.ptrs = &nv50_instobj_fast;
+		else
+			iobj->base.memory.ptrs = &nv50_instobj_slow;
+		refcount_inc(&iobj->maps);
+	}
+
+	mutex_unlock(&imem->subdev.mutex);
+	return map;
 }
 
 static void
 nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
 {
 	struct nv50_instobj *iobj = nv50_instobj(memory);
+	struct nvkm_instmem *imem = &iobj->imem->base;
+
+	mutex_lock(&imem->subdev.mutex);
 	nv50_instobj_kmap(iobj, vmm);
+	mutex_unlock(&imem->subdev.mutex);
 }
 
 static u64
@@ -177,12 +244,14 @@ static void *
 nv50_instobj_dtor(struct nvkm_memory *memory)
 {
 	struct nv50_instobj *iobj = nv50_instobj(memory);
-	struct nvkm_ram *ram = iobj->imem->base.subdev.device->fb->ram;
-	if (!IS_ERR_OR_NULL(iobj->map)) {
+	struct nvkm_instmem *imem = &iobj->imem->base;
+	struct nvkm_ram *ram = imem->subdev.device->fb->ram;
+	if (iobj->map) {
 		iounmap(iobj->map);
 		nvkm_vm_put(&iobj->bar);
 	}
 	ram->func->put(ram, &iobj->mem);
+	nvkm_instobj_dtor(imem, &iobj->base);
 	return iobj;
 }
 
@@ -209,11 +278,12 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
 
 	if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
 		return -ENOMEM;
-	*pmemory = &iobj->memory;
+	*pmemory = &iobj->base.memory;
 
-	nvkm_memory_ctor(&nv50_instobj_func, &iobj->memory);
-	iobj->memory.ptrs = &nv50_instobj_slow;
+	nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
+	iobj->base.memory.ptrs = &nv50_instobj_slow;
 	iobj->imem = imem;
+	refcount_set(&iobj->maps, 0);
 
 	size  = max((size  + 4095) & ~4095, (u32)4096);
 	align = max((align + 4095) & ~4095, (u32)4096);
@@ -240,7 +310,7 @@ static const struct nvkm_instmem_func
 nv50_instmem = {
 	.fini = nv50_instmem_fini,
 	.memory_new = nv50_instobj_new,
-	.persistent = false,
+	.persistent = true,
 	.zero = false,
 };