/* * Copyright 2018 Red Hat Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include "nouveau_svm.h" #include "nouveau_drv.h" #include "nouveau_chan.h" #include "nouveau_dmem.h" #include #include #include #include #include #include #include #include #include struct nouveau_svm { struct nouveau_drm *drm; struct mutex mutex; struct list_head inst; struct nouveau_svm_fault_buffer { int id; struct nvif_object object; u32 entries; u32 getaddr; u32 putaddr; u32 get; u32 put; struct nvif_notify notify; struct nouveau_svm_fault { u64 inst; u64 addr; u64 time; u32 engine; u8 gpc; u8 hub; u8 access; u8 client; u8 fault; struct nouveau_svmm *svmm; } **fault; int fault_nr; } buffer[1]; }; #define SVM_DBG(s,f,a...) NV_DEBUG((s)->drm, "svm: "f"\n", ##a) #define SVM_ERR(s,f,a...) NV_WARN((s)->drm, "svm: "f"\n", ##a) struct nouveau_pfnmap_args { struct nvif_ioctl_v0 i; struct nvif_ioctl_mthd_v0 m; struct nvif_vmm_pfnmap_v0 p; }; struct nouveau_ivmm { struct nouveau_svmm *svmm; u64 inst; struct list_head head; }; static struct nouveau_ivmm * nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst) { struct nouveau_ivmm *ivmm; list_for_each_entry(ivmm, &svm->inst, head) { if (ivmm->inst == inst) return ivmm; } return NULL; } struct nouveau_svmm { struct mmu_notifier notifier; struct nouveau_vmm *vmm; struct { unsigned long start; unsigned long limit; } unmanaged; struct mutex mutex; }; #define SVMM_DBG(s,f,a...) \ NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) #define SVMM_ERR(s,f,a...) \ NV_WARN((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) int nouveau_svmm_bind(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct nouveau_cli *cli = nouveau_cli(file_priv); struct drm_nouveau_svm_bind *args = data; unsigned target, cmd, priority; unsigned long addr, end, size; struct mm_struct *mm; args->va_start &= PAGE_MASK; args->va_end &= PAGE_MASK; /* Sanity check arguments */ if (args->reserved0 || args->reserved1) return -EINVAL; if (args->header & (~NOUVEAU_SVM_BIND_VALID_MASK)) return -EINVAL; if (args->va_start >= args->va_end) return -EINVAL; if (!args->npages) return -EINVAL; cmd = args->header >> NOUVEAU_SVM_BIND_COMMAND_SHIFT; cmd &= NOUVEAU_SVM_BIND_COMMAND_MASK; switch (cmd) { case NOUVEAU_SVM_BIND_COMMAND__MIGRATE: break; default: return -EINVAL; } priority = args->header >> NOUVEAU_SVM_BIND_PRIORITY_SHIFT; priority &= NOUVEAU_SVM_BIND_PRIORITY_MASK; /* FIXME support CPU target ie all target value < GPU_VRAM */ target = args->header >> NOUVEAU_SVM_BIND_TARGET_SHIFT; target &= NOUVEAU_SVM_BIND_TARGET_MASK; switch (target) { case NOUVEAU_SVM_BIND_TARGET__GPU_VRAM: break; default: return -EINVAL; } /* * FIXME: For now refuse non 0 stride, we need to change the migrate * kernel function to handle stride to avoid to create a mess within * each device driver. */ if (args->stride) return -EINVAL; size = ((unsigned long)args->npages) << PAGE_SHIFT; if ((args->va_start + size) <= args->va_start) return -EINVAL; if ((args->va_start + size) > args->va_end) return -EINVAL; /* * Ok we are ask to do something sane, for now we only support migrate * commands but we will add things like memory policy (what to do on * page fault) and maybe some other commands. */ mm = get_task_mm(current); mmap_read_lock(mm); if (!cli->svm.svmm) { mmap_read_unlock(mm); return -EINVAL; } for (addr = args->va_start, end = args->va_start + size; addr < end;) { struct vm_area_struct *vma; unsigned long next; vma = find_vma_intersection(mm, addr, end); if (!vma) break; addr = max(addr, vma->vm_start); next = min(vma->vm_end, end); /* This is a best effort so we ignore errors */ nouveau_dmem_migrate_vma(cli->drm, cli->svm.svmm, vma, addr, next); addr = next; } /* * FIXME Return the number of page we have migrated, again we need to * update the migrate API to return that information so that we can * report it to user space. */ args->result = 0; mmap_read_unlock(mm); mmput(mm); return 0; } /* Unlink channel instance from SVMM. */ void nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst) { struct nouveau_ivmm *ivmm; if (svmm) { mutex_lock(&svmm->vmm->cli->drm->svm->mutex); ivmm = nouveau_ivmm_find(svmm->vmm->cli->drm->svm, inst); if (ivmm) { list_del(&ivmm->head); kfree(ivmm); } mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); } } /* Link channel instance to SVMM. */ int nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst) { struct nouveau_ivmm *ivmm; if (svmm) { if (!(ivmm = kmalloc(sizeof(*ivmm), GFP_KERNEL))) return -ENOMEM; ivmm->svmm = svmm; ivmm->inst = inst; mutex_lock(&svmm->vmm->cli->drm->svm->mutex); list_add(&ivmm->head, &svmm->vmm->cli->drm->svm->inst); mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); } return 0; } /* Invalidate SVMM address-range on GPU. */ static void nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit) { if (limit > start) { bool super = svmm->vmm->vmm.object.client->super; svmm->vmm->vmm.object.client->super = true; nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR, &(struct nvif_vmm_pfnclr_v0) { .addr = start, .size = limit - start, }, sizeof(struct nvif_vmm_pfnclr_v0)); svmm->vmm->vmm.object.client->super = super; } } static int nouveau_svmm_invalidate_range_start(struct mmu_notifier *mn, const struct mmu_notifier_range *update) { struct nouveau_svmm *svmm = container_of(mn, struct nouveau_svmm, notifier); unsigned long start = update->start; unsigned long limit = update->end; if (!mmu_notifier_range_blockable(update)) return -EAGAIN; SVMM_DBG(svmm, "invalidate %016lx-%016lx", start, limit); mutex_lock(&svmm->mutex); if (unlikely(!svmm->vmm)) goto out; if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) { if (start < svmm->unmanaged.start) { nouveau_svmm_invalidate(svmm, start, svmm->unmanaged.limit); } start = svmm->unmanaged.limit; } nouveau_svmm_invalidate(svmm, start, limit); out: mutex_unlock(&svmm->mutex); return 0; } static void nouveau_svmm_free_notifier(struct mmu_notifier *mn) { kfree(container_of(mn, struct nouveau_svmm, notifier)); } static const struct mmu_notifier_ops nouveau_mn_ops = { .invalidate_range_start = nouveau_svmm_invalidate_range_start, .free_notifier = nouveau_svmm_free_notifier, }; void nouveau_svmm_fini(struct nouveau_svmm **psvmm) { struct nouveau_svmm *svmm = *psvmm; if (svmm) { mutex_lock(&svmm->mutex); svmm->vmm = NULL; mutex_unlock(&svmm->mutex); mmu_notifier_put(&svmm->notifier); *psvmm = NULL; } } int nouveau_svmm_init(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct nouveau_cli *cli = nouveau_cli(file_priv); struct nouveau_svmm *svmm; struct drm_nouveau_svm_init *args = data; int ret; /* Allocate tracking for SVM-enabled VMM. */ if (!(svmm = kzalloc(sizeof(*svmm), GFP_KERNEL))) return -ENOMEM; svmm->vmm = &cli->svm; svmm->unmanaged.start = args->unmanaged_addr; svmm->unmanaged.limit = args->unmanaged_addr + args->unmanaged_size; mutex_init(&svmm->mutex); /* Check that SVM isn't already enabled for the client. */ mutex_lock(&cli->mutex); if (cli->svm.cli) { ret = -EBUSY; goto out_free; } /* Allocate a new GPU VMM that can support SVM (managed by the * client, with replayable faults enabled). * * All future channel/memory allocations will make use of this * VMM instead of the standard one. */ ret = nvif_vmm_ctor(&cli->mmu, "svmVmm", cli->vmm.vmm.object.oclass, true, args->unmanaged_addr, args->unmanaged_size, &(struct gp100_vmm_v0) { .fault_replay = true, }, sizeof(struct gp100_vmm_v0), &cli->svm.vmm); if (ret) goto out_free; mmap_write_lock(current->mm); svmm->notifier.ops = &nouveau_mn_ops; ret = __mmu_notifier_register(&svmm->notifier, current->mm); if (ret) goto out_mm_unlock; /* Note, ownership of svmm transfers to mmu_notifier */ cli->svm.svmm = svmm; cli->svm.cli = cli; mmap_write_unlock(current->mm); mutex_unlock(&cli->mutex); return 0; out_mm_unlock: mmap_write_unlock(current->mm); out_free: mutex_unlock(&cli->mutex); kfree(svmm); return ret; } /* Issue fault replay for GPU to retry accesses that faulted previously. */ static void nouveau_svm_fault_replay(struct nouveau_svm *svm) { SVM_DBG(svm, "replay"); WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, GP100_VMM_VN_FAULT_REPLAY, &(struct gp100_vmm_fault_replay_vn) {}, sizeof(struct gp100_vmm_fault_replay_vn))); } /* Cancel a replayable fault that could not be handled. * * Cancelling the fault will trigger recovery to reset the engine * and kill the offending channel (ie. GPU SIGSEGV). */ static void nouveau_svm_fault_cancel(struct nouveau_svm *svm, u64 inst, u8 hub, u8 gpc, u8 client) { SVM_DBG(svm, "cancel %016llx %d %02x %02x", inst, hub, gpc, client); WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, GP100_VMM_VN_FAULT_CANCEL, &(struct gp100_vmm_fault_cancel_v0) { .hub = hub, .gpc = gpc, .client = client, .inst = inst, }, sizeof(struct gp100_vmm_fault_cancel_v0))); } static void nouveau_svm_fault_cancel_fault(struct nouveau_svm *svm, struct nouveau_svm_fault *fault) { nouveau_svm_fault_cancel(svm, fault->inst, fault->hub, fault->gpc, fault->client); } static int nouveau_svm_fault_cmp(const void *a, const void *b) { const struct nouveau_svm_fault *fa = *(struct nouveau_svm_fault **)a; const struct nouveau_svm_fault *fb = *(struct nouveau_svm_fault **)b; int ret; if ((ret = (s64)fa->inst - fb->inst)) return ret; if ((ret = (s64)fa->addr - fb->addr)) return ret; /*XXX: atomic? */ return (fa->access == 0 || fa->access == 3) - (fb->access == 0 || fb->access == 3); } static void nouveau_svm_fault_cache(struct nouveau_svm *svm, struct nouveau_svm_fault_buffer *buffer, u32 offset) { struct nvif_object *memory = &buffer->object; const u32 instlo = nvif_rd32(memory, offset + 0x00); const u32 insthi = nvif_rd32(memory, offset + 0x04); const u32 addrlo = nvif_rd32(memory, offset + 0x08); const u32 addrhi = nvif_rd32(memory, offset + 0x0c); const u32 timelo = nvif_rd32(memory, offset + 0x10); const u32 timehi = nvif_rd32(memory, offset + 0x14); const u32 engine = nvif_rd32(memory, offset + 0x18); const u32 info = nvif_rd32(memory, offset + 0x1c); const u64 inst = (u64)insthi << 32 | instlo; const u8 gpc = (info & 0x1f000000) >> 24; const u8 hub = (info & 0x00100000) >> 20; const u8 client = (info & 0x00007f00) >> 8; struct nouveau_svm_fault *fault; //XXX: i think we're supposed to spin waiting */ if (WARN_ON(!(info & 0x80000000))) return; nvif_mask(memory, offset + 0x1c, 0x80000000, 0x00000000); if (!buffer->fault[buffer->fault_nr]) { fault = kmalloc(sizeof(*fault), GFP_KERNEL); if (WARN_ON(!fault)) { nouveau_svm_fault_cancel(svm, inst, hub, gpc, client); return; } buffer->fault[buffer->fault_nr] = fault; } fault = buffer->fault[buffer->fault_nr++]; fault->inst = inst; fault->addr = (u64)addrhi << 32 | addrlo; fault->time = (u64)timehi << 32 | timelo; fault->engine = engine; fault->gpc = gpc; fault->hub = hub; fault->access = (info & 0x000f0000) >> 16; fault->client = client; fault->fault = (info & 0x0000001f); SVM_DBG(svm, "fault %016llx %016llx %02x", fault->inst, fault->addr, fault->access); } struct svm_notifier { struct mmu_interval_notifier notifier; struct nouveau_svmm *svmm; }; static bool nouveau_svm_range_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq) { struct svm_notifier *sn = container_of(mni, struct svm_notifier, notifier); /* * serializes the update to mni->invalidate_seq done by caller and * prevents invalidation of the PTE from progressing while HW is being * programmed. This is very hacky and only works because the normal * notifier that does invalidation is always called after the range * notifier. */ if (mmu_notifier_range_blockable(range)) mutex_lock(&sn->svmm->mutex); else if (!mutex_trylock(&sn->svmm->mutex)) return false; mmu_interval_set_seq(mni, cur_seq); mutex_unlock(&sn->svmm->mutex); return true; } static const struct mmu_interval_notifier_ops nouveau_svm_mni_ops = { .invalidate = nouveau_svm_range_invalidate, }; static void nouveau_hmm_convert_pfn(struct nouveau_drm *drm, struct hmm_range *range, u64 *ioctl_addr) { unsigned long i, npages; /* * The ioctl_addr prepared here is passed through nvif_object_ioctl() * to an eventual DMA map in something like gp100_vmm_pgt_pfn() * * This is all just encoding the internal hmm representation into a * different nouveau internal representation. */ npages = (range->end - range->start) >> PAGE_SHIFT; for (i = 0; i < npages; ++i) { struct page *page; if (!(range->hmm_pfns[i] & HMM_PFN_VALID)) { ioctl_addr[i] = 0; continue; } page = hmm_pfn_to_page(range->hmm_pfns[i]); if (is_device_private_page(page)) ioctl_addr[i] = nouveau_dmem_page_addr(page) | NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM; else ioctl_addr[i] = page_to_phys(page) | NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_HOST; if (range->hmm_pfns[i] & HMM_PFN_WRITE) ioctl_addr[i] |= NVIF_VMM_PFNMAP_V0_W; } } static int nouveau_range_fault(struct nouveau_svmm *svmm, struct nouveau_drm *drm, void *data, u32 size, unsigned long hmm_pfns[], u64 *ioctl_addr, struct svm_notifier *notifier) { unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); /* Have HMM fault pages within the fault window to the GPU. */ struct hmm_range range = { .notifier = ¬ifier->notifier, .start = notifier->notifier.interval_tree.start, .end = notifier->notifier.interval_tree.last + 1, .pfn_flags_mask = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE, .hmm_pfns = hmm_pfns, .dev_private_owner = drm->dev, }; struct mm_struct *mm = notifier->notifier.mm; int ret; while (true) { if (time_after(jiffies, timeout)) return -EBUSY; range.notifier_seq = mmu_interval_read_begin(range.notifier); mmap_read_lock(mm); ret = hmm_range_fault(&range); mmap_read_unlock(mm); if (ret) { /* * FIXME: the input PFN_REQ flags are destroyed on * -EBUSY, we need to regenerate them, also for the * other continue below */ if (ret == -EBUSY) continue; return ret; } mutex_lock(&svmm->mutex); if (mmu_interval_read_retry(range.notifier, range.notifier_seq)) { mutex_unlock(&svmm->mutex); continue; } break; } nouveau_hmm_convert_pfn(drm, &range, ioctl_addr); svmm->vmm->vmm.object.client->super = true; ret = nvif_object_ioctl(&svmm->vmm->vmm.object, data, size, NULL); svmm->vmm->vmm.object.client->super = false; mutex_unlock(&svmm->mutex); return ret; } static int nouveau_svm_fault(struct nvif_notify *notify) { struct nouveau_svm_fault_buffer *buffer = container_of(notify, typeof(*buffer), notify); struct nouveau_svm *svm = container_of(buffer, typeof(*svm), buffer[buffer->id]); struct nvif_object *device = &svm->drm->client.device.object; struct nouveau_svmm *svmm; struct { struct { struct nvif_ioctl_v0 i; struct nvif_ioctl_mthd_v0 m; struct nvif_vmm_pfnmap_v0 p; } i; u64 phys[16]; } args; unsigned long hmm_pfns[ARRAY_SIZE(args.phys)]; struct vm_area_struct *vma; u64 inst, start, limit; int fi, fn, pi, fill; int replay = 0, ret; /* Parse available fault buffer entries into a cache, and update * the GET pointer so HW can reuse the entries. */ SVM_DBG(svm, "fault handler"); if (buffer->get == buffer->put) { buffer->put = nvif_rd32(device, buffer->putaddr); buffer->get = nvif_rd32(device, buffer->getaddr); if (buffer->get == buffer->put) return NVIF_NOTIFY_KEEP; } buffer->fault_nr = 0; SVM_DBG(svm, "get %08x put %08x", buffer->get, buffer->put); while (buffer->get != buffer->put) { nouveau_svm_fault_cache(svm, buffer, buffer->get * 0x20); if (++buffer->get == buffer->entries) buffer->get = 0; } nvif_wr32(device, buffer->getaddr, buffer->get); SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr); /* Sort parsed faults by instance pointer to prevent unnecessary * instance to SVMM translations, followed by address and access * type to reduce the amount of work when handling the faults. */ sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault), nouveau_svm_fault_cmp, NULL); /* Lookup SVMM structure for each unique instance pointer. */ mutex_lock(&svm->mutex); for (fi = 0, svmm = NULL; fi < buffer->fault_nr; fi++) { if (!svmm || buffer->fault[fi]->inst != inst) { struct nouveau_ivmm *ivmm = nouveau_ivmm_find(svm, buffer->fault[fi]->inst); svmm = ivmm ? ivmm->svmm : NULL; inst = buffer->fault[fi]->inst; SVM_DBG(svm, "inst %016llx -> svm-%p", inst, svmm); } buffer->fault[fi]->svmm = svmm; } mutex_unlock(&svm->mutex); /* Process list of faults. */ args.i.i.version = 0; args.i.i.type = NVIF_IOCTL_V0_MTHD; args.i.m.version = 0; args.i.m.method = NVIF_VMM_V0_PFNMAP; args.i.p.version = 0; for (fi = 0; fn = fi + 1, fi < buffer->fault_nr; fi = fn) { struct svm_notifier notifier; struct mm_struct *mm; /* Cancel any faults from non-SVM channels. */ if (!(svmm = buffer->fault[fi]->svmm)) { nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); continue; } SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr); /* We try and group handling of faults within a small * window into a single update. */ start = buffer->fault[fi]->addr; limit = start + (ARRAY_SIZE(args.phys) << PAGE_SHIFT); if (start < svmm->unmanaged.limit) limit = min_t(u64, limit, svmm->unmanaged.start); SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); mm = svmm->notifier.mm; if (!mmget_not_zero(mm)) { nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); continue; } /* Intersect fault window with the CPU VMA, cancelling * the fault if the address is invalid. */ mmap_read_lock(mm); vma = find_vma_intersection(mm, start, limit); if (!vma) { SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit); mmap_read_unlock(mm); mmput(mm); nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); continue; } start = max_t(u64, start, vma->vm_start); limit = min_t(u64, limit, vma->vm_end); mmap_read_unlock(mm); SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); if (buffer->fault[fi]->addr != start) { SVMM_ERR(svmm, "addr %016llx", buffer->fault[fi]->addr); mmput(mm); nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); continue; } /* Prepare the GPU-side update of all pages within the * fault window, determining required pages and access * permissions based on pending faults. */ args.i.p.page = PAGE_SHIFT; args.i.p.addr = start; for (fn = fi, pi = 0;;) { /* Determine required permissions based on GPU fault * access flags. *XXX: atomic? */ switch (buffer->fault[fn]->access) { case 0: /* READ. */ hmm_pfns[pi++] = HMM_PFN_REQ_FAULT; break; case 3: /* PREFETCH. */ hmm_pfns[pi++] = 0; break; default: hmm_pfns[pi++] = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE; break; } args.i.p.size = pi << PAGE_SHIFT; /* It's okay to skip over duplicate addresses from the * same SVMM as faults are ordered by access type such * that only the first one needs to be handled. * * ie. WRITE faults appear first, thus any handling of * pending READ faults will already be satisfied. */ while (++fn < buffer->fault_nr && buffer->fault[fn]->svmm == svmm && buffer->fault[fn ]->addr == buffer->fault[fn - 1]->addr); /* If the next fault is outside the window, or all GPU * faults have been dealt with, we're done here. */ if (fn >= buffer->fault_nr || buffer->fault[fn]->svmm != svmm || buffer->fault[fn]->addr >= limit) break; /* Fill in the gap between this fault and the next. */ fill = (buffer->fault[fn ]->addr - buffer->fault[fn - 1]->addr) >> PAGE_SHIFT; while (--fill) hmm_pfns[pi++] = 0; } SVMM_DBG(svmm, "wndw %016llx-%016llx covering %d fault(s)", args.i.p.addr, args.i.p.addr + args.i.p.size, fn - fi); notifier.svmm = svmm; ret = mmu_interval_notifier_insert(¬ifier.notifier, svmm->notifier.mm, args.i.p.addr, args.i.p.size, &nouveau_svm_mni_ops); if (!ret) { ret = nouveau_range_fault( svmm, svm->drm, &args, sizeof(args.i) + pi * sizeof(args.phys[0]), hmm_pfns, args.phys, ¬ifier); mmu_interval_notifier_remove(¬ifier.notifier); } mmput(mm); /* Cancel any faults in the window whose pages didn't manage * to keep their valid bit, or stay writeable when required. * * If handling failed completely, cancel all faults. */ while (fi < fn) { struct nouveau_svm_fault *fault = buffer->fault[fi++]; pi = (fault->addr - args.i.p.addr) >> PAGE_SHIFT; if (ret || !(args.phys[pi] & NVIF_VMM_PFNMAP_V0_V) || (!(args.phys[pi] & NVIF_VMM_PFNMAP_V0_W) && fault->access != 0 && fault->access != 3)) { nouveau_svm_fault_cancel_fault(svm, fault); continue; } replay++; } } /* Issue fault replay to the GPU. */ if (replay) nouveau_svm_fault_replay(svm); return NVIF_NOTIFY_KEEP; } static struct nouveau_pfnmap_args * nouveau_pfns_to_args(void *pfns) { return container_of(pfns, struct nouveau_pfnmap_args, p.phys); } u64 * nouveau_pfns_alloc(unsigned long npages) { struct nouveau_pfnmap_args *args; args = kzalloc(struct_size(args, p.phys, npages), GFP_KERNEL); if (!args) return NULL; args->i.type = NVIF_IOCTL_V0_MTHD; args->m.method = NVIF_VMM_V0_PFNMAP; args->p.page = PAGE_SHIFT; return args->p.phys; } void nouveau_pfns_free(u64 *pfns) { struct nouveau_pfnmap_args *args = nouveau_pfns_to_args(pfns); kfree(args); } void nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm, unsigned long addr, u64 *pfns, unsigned long npages) { struct nouveau_pfnmap_args *args = nouveau_pfns_to_args(pfns); int ret; args->p.addr = addr; args->p.size = npages << PAGE_SHIFT; mutex_lock(&svmm->mutex); svmm->vmm->vmm.object.client->super = true; ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, sizeof(*args) + npages * sizeof(args->p.phys[0]), NULL); svmm->vmm->vmm.object.client->super = false; mutex_unlock(&svmm->mutex); } static void nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id) { struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; nvif_notify_put(&buffer->notify); } static int nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id) { struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; struct nvif_object *device = &svm->drm->client.device.object; buffer->get = nvif_rd32(device, buffer->getaddr); buffer->put = nvif_rd32(device, buffer->putaddr); SVM_DBG(svm, "get %08x put %08x (init)", buffer->get, buffer->put); return nvif_notify_get(&buffer->notify); } static void nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id) { struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; int i; if (buffer->fault) { for (i = 0; buffer->fault[i] && i < buffer->entries; i++) kfree(buffer->fault[i]); kvfree(buffer->fault); } nouveau_svm_fault_buffer_fini(svm, id); nvif_notify_dtor(&buffer->notify); nvif_object_dtor(&buffer->object); } static int nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id) { struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; struct nouveau_drm *drm = svm->drm; struct nvif_object *device = &drm->client.device.object; struct nvif_clb069_v0 args = {}; int ret; buffer->id = id; ret = nvif_object_ctor(device, "svmFaultBuffer", 0, oclass, &args, sizeof(args), &buffer->object); if (ret < 0) { SVM_ERR(svm, "Fault buffer allocation failed: %d", ret); return ret; } nvif_object_map(&buffer->object, NULL, 0); buffer->entries = args.entries; buffer->getaddr = args.get; buffer->putaddr = args.put; ret = nvif_notify_ctor(&buffer->object, "svmFault", nouveau_svm_fault, true, NVB069_V0_NTFY_FAULT, NULL, 0, 0, &buffer->notify); if (ret) return ret; buffer->fault = kvzalloc(sizeof(*buffer->fault) * buffer->entries, GFP_KERNEL); if (!buffer->fault) return -ENOMEM; return nouveau_svm_fault_buffer_init(svm, id); } void nouveau_svm_resume(struct nouveau_drm *drm) { struct nouveau_svm *svm = drm->svm; if (svm) nouveau_svm_fault_buffer_init(svm, 0); } void nouveau_svm_suspend(struct nouveau_drm *drm) { struct nouveau_svm *svm = drm->svm; if (svm) nouveau_svm_fault_buffer_fini(svm, 0); } void nouveau_svm_fini(struct nouveau_drm *drm) { struct nouveau_svm *svm = drm->svm; if (svm) { nouveau_svm_fault_buffer_dtor(svm, 0); kfree(drm->svm); drm->svm = NULL; } } void nouveau_svm_init(struct nouveau_drm *drm) { static const struct nvif_mclass buffers[] = { { VOLTA_FAULT_BUFFER_A, 0 }, { MAXWELL_FAULT_BUFFER_A, 0 }, {} }; struct nouveau_svm *svm; int ret; /* Disable on Volta and newer until channel recovery is fixed, * otherwise clients will have a trivial way to trash the GPU * for everyone. */ if (drm->client.device.info.family > NV_DEVICE_INFO_V0_PASCAL) return; if (!(drm->svm = svm = kzalloc(sizeof(*drm->svm), GFP_KERNEL))) return; drm->svm->drm = drm; mutex_init(&drm->svm->mutex); INIT_LIST_HEAD(&drm->svm->inst); ret = nvif_mclass(&drm->client.device.object, buffers); if (ret < 0) { SVM_DBG(svm, "No supported fault buffer class"); nouveau_svm_fini(drm); return; } ret = nouveau_svm_fault_buffer_ctor(svm, buffers[ret].oclass, 0); if (ret) { nouveau_svm_fini(drm); return; } SVM_DBG(svm, "Initialised"); }