2 * Copyright 2017 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 #define NVKM_VMM_LEVELS_MAX 5
25 #include <subdev/fb.h>
28 nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
30 struct nvkm_vmm_pt *pgt = *ppgt;
39 static struct nvkm_vmm_pt *
40 nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
41 const struct nvkm_vmm_page *page)
43 const u32 pten = 1 << desc->bits;
44 struct nvkm_vmm_pt *pgt;
47 if (desc->type > PGT) {
48 if (desc->type == SPT) {
49 const struct nvkm_vmm_desc *pair = page[-1].desc;
50 lpte = pten >> (desc->bits - pair->bits);
56 if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
58 pgt->page = page ? page->shift : 0;
61 if (desc->type == PGD) {
62 pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
72 struct nvkm_vmm_iter {
73 const struct nvkm_vmm_page *page;
74 const struct nvkm_vmm_desc *desc;
78 u32 pte[NVKM_VMM_LEVELS_MAX];
79 struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
83 #ifdef CONFIG_NOUVEAU_DEBUG_MMU
85 nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
88 case PGD: return "PGD";
89 case PGT: return "PGT";
90 case SPT: return "SPT";
91 case LPT: return "LPT";
98 nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
101 for (lvl = it->max; lvl >= 0; lvl--) {
103 buf += sprintf(buf, "%05x:", it->pte[lvl]);
105 buf += sprintf(buf, "xxxxx:");
109 #define TRA(i,f,a...) do { \
110 char _buf[NVKM_VMM_LEVELS_MAX * 7]; \
111 struct nvkm_vmm_iter *_it = (i); \
112 nvkm_vmm_trace(_it, _buf); \
113 VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \
116 #define TRA(i,f,a...)
120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
122 it->flush = min(it->flush, it->max - it->lvl);
126 nvkm_vmm_flush(struct nvkm_vmm_iter *it)
128 if (it->flush != NVKM_VMM_LEVELS_MAX) {
129 if (it->vmm->func->flush) {
130 TRA(it, "flush: %d", it->flush);
131 it->vmm->func->flush(it->vmm, it->flush);
133 it->flush = NVKM_VMM_LEVELS_MAX;
138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
140 const struct nvkm_vmm_desc *desc = it->desc;
141 const int type = desc[it->lvl].type == SPT;
142 struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
143 struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
144 struct nvkm_mmu_pt *pt = pgt->pt[type];
145 struct nvkm_vmm *vmm = it->vmm;
146 u32 pdei = it->pte[it->lvl + 1];
148 /* Recurse up the tree, unreferencing/destroying unneeded PDs. */
150 if (--pgd->refs[0]) {
151 const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
152 /* PD has other valid PDEs, so we need a proper update. */
153 TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
154 pgt->pt[type] = NULL;
155 if (!pgt->refs[!type]) {
156 /* PDE no longer required. */
159 func->sparse(vmm, pgd->pt[0], pdei, 1);
160 pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
162 func->unmap(vmm, pgd->pt[0], pdei, 1);
163 pgd->pde[pdei] = NULL;
166 /* Special handling for Tesla-class GPUs,
167 * where there's no central PD, but each
168 * instance has its own embedded PD.
170 func->pde(vmm, pgd, pdei);
171 pgd->pde[pdei] = NULL;
174 /* PDE was pointing at dual-PTs and we're removing
175 * one of them, leaving the other in place.
177 func->pde(vmm, pgd, pdei);
180 /* GPU may have cached the PTs, flush before freeing. */
181 nvkm_vmm_flush_mark(it);
184 /* PD has no valid PDEs left, so we can just destroy it. */
185 nvkm_vmm_unref_pdes(it);
189 TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
190 nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
191 if (!pgt->refs[!type])
192 nvkm_vmm_pt_del(&pgt);
197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
198 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
200 const struct nvkm_vmm_desc *pair = it->page[-1].desc;
201 const u32 sptb = desc->bits - pair->bits;
202 const u32 sptn = 1 << sptb;
203 struct nvkm_vmm *vmm = it->vmm;
204 u32 spti = ptei & (sptn - 1), lpti, pteb;
206 /* Determine how many SPTEs are being touched under each LPTE,
207 * and drop reference counts.
209 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
210 const u32 pten = min(sptn - spti, ptes);
211 pgt->pte[lpti] -= pten;
215 /* We're done here if there's no corresponding LPT. */
219 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
220 /* Skip over any LPTEs that still have valid SPTEs. */
221 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
222 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
223 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
229 /* As there's no more non-UNMAPPED SPTEs left in the range
230 * covered by a number of LPTEs, the LPTEs once again take
231 * control over their address range.
233 * Determine how many LPTEs need to transition state.
235 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
236 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
237 if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
239 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
242 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
243 TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
244 pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
246 if (pair->func->invalid) {
247 /* If the MMU supports it, restore the LPTE to the
248 * INVALID state to tell the MMU there is no point
249 * trying to fetch the corresponding SPTEs.
251 TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
252 pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
260 const struct nvkm_vmm_desc *desc = it->desc;
261 const int type = desc->type == SPT;
262 struct nvkm_vmm_pt *pgt = it->pt[0];
264 /* Drop PTE references. */
265 pgt->refs[type] -= ptes;
267 /* Dual-PTs need special handling, unless PDE becoming invalid. */
268 if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
269 nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
271 /* PT no longer neeed? Destroy it. */
272 if (!pgt->refs[type]) {
274 TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
276 nvkm_vmm_unref_pdes(it);
277 return false; /* PTE writes for unmap() not necessary. */
284 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
285 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
287 const struct nvkm_vmm_desc *pair = it->page[-1].desc;
288 const u32 sptb = desc->bits - pair->bits;
289 const u32 sptn = 1 << sptb;
290 struct nvkm_vmm *vmm = it->vmm;
291 u32 spti = ptei & (sptn - 1), lpti, pteb;
293 /* Determine how many SPTEs are being touched under each LPTE,
294 * and increase reference counts.
296 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
297 const u32 pten = min(sptn - spti, ptes);
298 pgt->pte[lpti] += pten;
302 /* We're done here if there's no corresponding LPT. */
306 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
307 /* Skip over any LPTEs that already have valid SPTEs. */
308 if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
309 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
310 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
316 /* As there are now non-UNMAPPED SPTEs in the range covered
317 * by a number of LPTEs, we need to transfer control of the
318 * address range to the SPTEs.
320 * Determine how many LPTEs need to transition state.
322 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
323 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
324 if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
326 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
329 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
330 const u32 spti = pteb * sptn;
331 const u32 sptc = ptes * sptn;
332 /* The entire LPTE is marked as sparse, we need
333 * to make sure that the SPTEs are too.
335 TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
336 desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
337 /* Sparse LPTEs prevent SPTEs from being accessed. */
338 TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
339 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
341 if (pair->func->invalid) {
342 /* MMU supports blocking SPTEs by marking an LPTE
343 * as INVALID. We need to reverse that here.
345 TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
346 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
352 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
354 const struct nvkm_vmm_desc *desc = it->desc;
355 const int type = desc->type == SPT;
356 struct nvkm_vmm_pt *pgt = it->pt[0];
358 /* Take PTE references. */
359 pgt->refs[type] += ptes;
361 /* Dual-PTs need special handling. */
362 if (desc->type == SPT)
363 nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
369 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
370 struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
372 if (desc->type == PGD) {
374 pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
376 if (desc->type == LPT) {
377 memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
382 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
384 struct nvkm_vmm_pt *pt = it->pt[0];
385 if (it->desc->type == PGD)
386 memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
388 if (it->desc->type == LPT)
389 memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
390 return nvkm_vmm_unref_ptes(it, ptei, ptes);
394 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
396 nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
397 return nvkm_vmm_ref_ptes(it, ptei, ptes);
401 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
403 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
404 const int type = desc->type == SPT;
405 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
406 const bool zero = !pgt->sparse && !desc->func->invalid;
407 struct nvkm_vmm *vmm = it->vmm;
408 struct nvkm_mmu *mmu = vmm->mmu;
409 struct nvkm_mmu_pt *pt;
410 u32 pten = 1 << desc->bits;
411 u32 pteb, ptei, ptes;
412 u32 size = desc->size * pten;
416 pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
417 if (!pgt->pt[type]) {
419 nvkm_vmm_unref_pdes(it);
428 if (desc->type == LPT && pgt->refs[1]) {
429 /* SPT already exists covering the same range as this LPT,
430 * which means we need to be careful that any LPTEs which
431 * overlap valid SPTEs are unmapped as opposed to invalid
432 * or sparse, which would prevent the MMU from looking at
433 * the SPTEs on some GPUs.
435 for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
436 bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
437 for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
438 bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
445 desc->func->sparse(vmm, pt, pteb, ptes);
447 desc->func->invalid(vmm, pt, pteb, ptes);
448 memset(&pgt->pte[pteb], 0x00, ptes);
450 desc->func->unmap(vmm, pt, pteb, ptes);
452 pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
457 nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
458 desc->func->sparse(vmm, pt, 0, pten);
460 desc->func->invalid(vmm, pt, 0, pten);
465 TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
466 it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
467 nvkm_vmm_flush_mark(it);
472 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
474 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
475 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
477 pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
480 nvkm_vmm_unref_pdes(it);
484 pgd->pde[pdei] = pgt;
489 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
490 u64 addr, u64 size, const char *name, bool ref,
491 bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32),
492 nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
493 nvkm_vmm_pxe_func CLR_PTES)
495 const struct nvkm_vmm_desc *desc = page->desc;
496 struct nvkm_vmm_iter it;
497 u64 bits = addr >> page->shift;
502 it.cnt = size >> page->shift;
503 it.flush = NVKM_VMM_LEVELS_MAX;
505 /* Deconstruct address into PTE indices for each mapping level. */
506 for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
507 it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
508 bits >>= desc[it.lvl].bits;
511 it.pt[it.max] = vmm->pd;
514 TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
515 addr, size, page->shift, it.cnt);
518 /* Depth-first traversal of page tables. */
520 struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
521 const int type = desc->type == SPT;
522 const u32 pten = 1 << desc->bits;
523 const u32 ptei = it.pte[0];
524 const u32 ptes = min_t(u64, it.cnt, pten - ptei);
526 /* Walk down the tree, finding page tables for each level. */
527 for (; it.lvl; it.lvl--) {
528 const u32 pdei = it.pte[it.lvl];
529 struct nvkm_vmm_pt *pgd = pgt;
532 if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
533 if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
536 it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
540 * This is a separate step from above due to GF100 and
541 * newer having dual page tables at some levels, which
542 * are refcounted independently.
544 if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
545 if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
550 /* Handle PTE updates. */
551 if (!REF_PTES || REF_PTES(&it, ptei, ptes)) {
552 struct nvkm_mmu_pt *pt = pgt->pt[type];
553 if (MAP_PTES || CLR_PTES) {
555 MAP_PTES(vmm, pt, ptei, ptes, map);
557 CLR_PTES(vmm, pt, ptei, ptes);
558 nvkm_vmm_flush_mark(&it);
562 /* Walk back up the tree to the next position. */
563 it.pte[it.lvl] += ptes;
566 while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
567 it.pte[it.lvl++] = 0;
577 /* Reconstruct the failure address so the caller is able to
578 * reverse any partially completed operations.
580 addr = it.pte[it.max--];
582 addr = addr << desc[it.max].bits;
583 addr |= it.pte[it.max];
586 return addr << page->shift;
590 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
593 nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false,
594 nvkm_vmm_sparse_unref_ptes, NULL, NULL,
595 page->desc->func->invalid ?
596 page->desc->func->invalid : page->desc->func->unmap);
600 nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
603 if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
604 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
605 true, nvkm_vmm_sparse_ref_ptes, NULL,
606 NULL, page->desc->func->sparse);
608 if ((size = fail - addr))
609 nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
618 nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
620 const struct nvkm_vmm_page *page = vmm->func->page;
626 /* Limit maximum page size based on remaining size. */
627 while (size < (1ULL << page[m].shift))
631 /* Find largest page size suitable for alignment. */
632 while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
635 /* Determine number of PTEs at this page size. */
637 /* Limited to alignment boundary of next page size. */
638 u64 next = 1ULL << page[i - 1].shift;
639 u64 part = ALIGN(addr, next) - addr;
640 if (size - part >= next)
641 block = (part >> page[i].shift) << page[i].shift;
643 block = (size >> page[i].shift) << page[i].shift;
645 block = (size >> page[i].shift) << page[i].shift;
648 /* Perform operation. */
650 int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
652 if ((size = addr - start))
653 nvkm_vmm_ptes_sparse(vmm, start, size, false);
657 nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
668 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
669 u64 addr, u64 size, bool sparse)
671 const struct nvkm_vmm_desc_func *func = page->desc->func;
672 nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
673 false, nvkm_vmm_unref_ptes, NULL, NULL,
674 sparse ? func->sparse : func->invalid ? func->invalid :
679 nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
680 u64 addr, u64 size, struct nvkm_vmm_map *map,
681 nvkm_vmm_pte_func func)
683 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
684 nvkm_vmm_ref_ptes, func, map, NULL);
686 if ((size = fail - addr))
687 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false);
694 nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
695 u64 addr, u64 size, bool sparse)
697 const struct nvkm_vmm_desc_func *func = page->desc->func;
698 nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL,
699 sparse ? func->sparse : func->invalid ? func->invalid :
704 nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
705 u64 addr, u64 size, struct nvkm_vmm_map *map,
706 nvkm_vmm_pte_func func)
708 nvkm_vmm_iter(vmm, page, addr, size, "map", false,
709 NULL, func, map, NULL);
713 nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
716 nvkm_vmm_iter(vmm, page, addr, size, "unref", false,
717 nvkm_vmm_unref_ptes, NULL, NULL, NULL);
721 nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
724 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true,
725 nvkm_vmm_ref_ptes, NULL, NULL, NULL);
728 nvkm_vmm_ptes_put(vmm, page, addr, fail - addr);
734 static inline struct nvkm_vma *
735 nvkm_vma_new(u64 addr, u64 size)
737 struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
741 vma->page = NVKM_VMA_PAGE_NONE;
742 vma->refd = NVKM_VMA_PAGE_NONE;
748 nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
750 struct nvkm_vma *new;
752 BUG_ON(vma->size == tail);
754 if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
758 new->mapref = vma->mapref;
759 new->sparse = vma->sparse;
760 new->page = vma->page;
761 new->refd = vma->refd;
762 new->used = vma->used;
763 new->part = vma->part;
764 new->user = vma->user;
765 new->busy = vma->busy;
766 list_add(&new->head, &vma->head);
771 nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
773 struct rb_node **ptr = &vmm->free.rb_node;
774 struct rb_node *parent = NULL;
777 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
779 if (vma->size < this->size)
780 ptr = &parent->rb_left;
782 if (vma->size > this->size)
783 ptr = &parent->rb_right;
785 if (vma->addr < this->addr)
786 ptr = &parent->rb_left;
788 if (vma->addr > this->addr)
789 ptr = &parent->rb_right;
794 rb_link_node(&vma->tree, parent, ptr);
795 rb_insert_color(&vma->tree, &vmm->free);
799 nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
801 struct rb_node **ptr = &vmm->root.rb_node;
802 struct rb_node *parent = NULL;
805 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
807 if (vma->addr < this->addr)
808 ptr = &parent->rb_left;
810 if (vma->addr > this->addr)
811 ptr = &parent->rb_right;
816 rb_link_node(&vma->tree, parent, ptr);
817 rb_insert_color(&vma->tree, &vmm->root);
821 nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
823 struct rb_node *node = vmm->root.rb_node;
825 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
826 if (addr < vma->addr)
827 node = node->rb_left;
829 if (addr >= vma->addr + vma->size)
830 node = node->rb_right;
838 nvkm_vmm_dtor(struct nvkm_vmm *vmm)
840 struct nvkm_vma *vma;
841 struct rb_node *node;
843 while ((node = rb_first(&vmm->root))) {
844 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
845 nvkm_vmm_put(vmm, &vma);
848 if (vmm->bootstrapped) {
849 const struct nvkm_vmm_page *page = vmm->func->page;
850 const u64 limit = vmm->limit - vmm->start;
852 while (page[1].shift)
855 nvkm_mmu_ptc_dump(vmm->mmu);
856 nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
859 vma = list_first_entry(&vmm->list, typeof(*vma), head);
860 list_del(&vma->head);
862 WARN_ON(!list_empty(&vmm->list));
865 dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
866 vmm->nullp, vmm->null);
870 nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
871 nvkm_vmm_pt_del(&vmm->pd);
876 nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
877 u32 pd_header, u64 addr, u64 size, struct lock_class_key *key,
878 const char *name, struct nvkm_vmm *vmm)
880 static struct lock_class_key _key;
881 const struct nvkm_vmm_page *page = func->page;
882 const struct nvkm_vmm_desc *desc;
883 struct nvkm_vma *vma;
884 int levels, bits = 0;
889 vmm->debug = mmu->subdev.debug;
890 kref_init(&vmm->kref);
892 __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key);
894 /* Locate the smallest page size supported by the backend, it will
895 * have the the deepest nesting of page tables.
897 while (page[1].shift)
900 /* Locate the structure that describes the layout of the top-level
901 * page table, and determine the number of valid bits in a virtual
904 for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
909 if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
913 vmm->limit = size ? (addr + size) : (1ULL << bits);
914 if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
917 /* Allocate top-level page table. */
918 vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
921 vmm->pd->refs[0] = 1;
922 INIT_LIST_HEAD(&vmm->join);
924 /* ... and the GPU storage for it, except on Tesla-class GPUs that
925 * have the PD embedded in the instance structure.
928 const u32 size = pd_header + desc->size * (1 << desc->bits);
929 vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
934 /* Initialise address-space MM. */
935 INIT_LIST_HEAD(&vmm->list);
939 if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
942 nvkm_vmm_free_insert(vmm, vma);
943 list_add(&vma->head, &vmm->list);
948 nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
949 u32 hdr, u64 addr, u64 size, struct lock_class_key *key,
950 const char *name, struct nvkm_vmm **pvmm)
952 if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
954 return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm);
957 #define node(root, dir) ((root)->head.dir == &vmm->list) ? NULL : \
958 list_entry((root)->head.dir, struct nvkm_vma, head)
961 nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
963 struct nvkm_vma *next;
965 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
966 nvkm_memory_unref(&vma->memory);
969 struct nvkm_vma *prev = node(vma, prev);
971 prev->size += vma->size;
972 rb_erase(&vma->tree, &vmm->root);
973 list_del(&vma->head);
979 next = node(vma, next);
980 if (next && next->part) {
982 vma->size += next->size;
983 rb_erase(&next->tree, &vmm->root);
984 list_del(&next->head);
991 nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
993 const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
996 nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse);
997 vma->refd = NVKM_VMA_PAGE_NONE;
999 nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse);
1002 nvkm_vmm_unmap_region(vmm, vma);
1006 nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1009 mutex_lock(&vmm->mutex);
1010 nvkm_vmm_unmap_locked(vmm, vma);
1011 mutex_unlock(&vmm->mutex);
1016 nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1017 void *argv, u32 argc, struct nvkm_vmm_map *map)
1019 switch (nvkm_memory_target(map->memory)) {
1020 case NVKM_MEM_TARGET_VRAM:
1021 if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1022 VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1026 case NVKM_MEM_TARGET_HOST:
1027 case NVKM_MEM_TARGET_NCOH:
1028 if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1029 VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1038 if (!IS_ALIGNED( vma->addr, 1ULL << map->page->shift) ||
1039 !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1040 !IS_ALIGNED( map->offset, 1ULL << map->page->shift) ||
1041 nvkm_memory_page(map->memory) < map->page->shift) {
1042 VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1043 vma->addr, (u64)vma->size, map->offset, map->page->shift,
1044 nvkm_memory_page(map->memory));
1048 return vmm->func->valid(vmm, argv, argc, map);
1052 nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1053 void *argv, u32 argc, struct nvkm_vmm_map *map)
1055 for (map->page = vmm->func->page; map->page->shift; map->page++) {
1056 VMM_DEBUG(vmm, "trying %d", map->page->shift);
1057 if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1064 nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1065 void *argv, u32 argc, struct nvkm_vmm_map *map)
1067 nvkm_vmm_pte_func func;
1070 /* Make sure we won't overrun the end of the memory object. */
1071 if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1072 VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1073 nvkm_memory_size(map->memory),
1074 map->offset, (u64)vma->size);
1078 /* Check remaining arguments for validity. */
1079 if (vma->page == NVKM_VMA_PAGE_NONE &&
1080 vma->refd == NVKM_VMA_PAGE_NONE) {
1081 /* Find the largest page size we can perform the mapping at. */
1082 const u32 debug = vmm->debug;
1084 ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1087 VMM_DEBUG(vmm, "invalid at any page size");
1088 nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1092 /* Page size of the VMA is already pre-determined. */
1093 if (vma->refd != NVKM_VMA_PAGE_NONE)
1094 map->page = &vmm->func->page[vma->refd];
1096 map->page = &vmm->func->page[vma->page];
1098 ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1100 VMM_DEBUG(vmm, "invalid %d\n", ret);
1105 /* Deal with the 'offset' argument, and fetch the backend function. */
1106 map->off = map->offset;
1108 for (; map->off; map->mem = map->mem->next) {
1109 u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1110 if (size > map->off)
1114 func = map->page->desc->func->mem;
1117 for (; map->off; map->sgl = sg_next(map->sgl)) {
1118 u64 size = sg_dma_len(map->sgl);
1119 if (size > map->off)
1123 func = map->page->desc->func->sgl;
1125 map->dma += map->offset >> PAGE_SHIFT;
1126 map->off = map->offset & PAGE_MASK;
1127 func = map->page->desc->func->dma;
1130 /* Perform the map. */
1131 if (vma->refd == NVKM_VMA_PAGE_NONE) {
1132 ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1136 vma->refd = map->page - vmm->func->page;
1138 nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1141 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1142 nvkm_memory_unref(&vma->memory);
1143 vma->memory = nvkm_memory_ref(map->memory);
1144 vma->tags = map->tags;
1149 nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1150 struct nvkm_vmm_map *map)
1153 mutex_lock(&vmm->mutex);
1154 ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1156 mutex_unlock(&vmm->mutex);
1161 nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1163 struct nvkm_vma *prev, *next;
1165 if ((prev = node(vma, prev)) && !prev->used) {
1166 rb_erase(&prev->tree, &vmm->free);
1167 list_del(&prev->head);
1168 vma->addr = prev->addr;
1169 vma->size += prev->size;
1173 if ((next = node(vma, next)) && !next->used) {
1174 rb_erase(&next->tree, &vmm->free);
1175 list_del(&next->head);
1176 vma->size += next->size;
1180 nvkm_vmm_free_insert(vmm, vma);
1184 nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1186 const struct nvkm_vmm_page *page = vmm->func->page;
1187 struct nvkm_vma *next = vma;
1191 if (vma->mapref || !vma->sparse) {
1193 const bool map = next->memory != NULL;
1194 const u8 refd = next->refd;
1195 const u64 addr = next->addr;
1196 u64 size = next->size;
1198 /* Merge regions that are in the same state. */
1199 while ((next = node(next, next)) && next->part &&
1200 (next->memory != NULL) == map &&
1201 (next->refd == refd))
1205 /* Region(s) are mapped, merge the unmap
1206 * and dereference into a single walk of
1209 nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1212 if (refd != NVKM_VMA_PAGE_NONE) {
1213 /* Drop allocation-time PTE references. */
1214 nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1216 } while (next && next->part);
1219 /* Merge any mapped regions that were split from the initial
1220 * address-space allocation back into the allocated VMA, and
1221 * release memory/compression resources.
1226 nvkm_vmm_unmap_region(vmm, next);
1227 } while ((next = node(vma, next)) && next->part);
1229 if (vma->sparse && !vma->mapref) {
1230 /* Sparse region that was allocated with a fixed page size,
1231 * meaning all relevant PTEs were referenced once when the
1232 * region was allocated, and remained that way, regardless
1233 * of whether memory was mapped into it afterwards.
1235 * The process of unmapping, unsparsing, and dereferencing
1236 * PTEs can be done in a single page tree walk.
1238 nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1241 /* Sparse region that wasn't allocated with a fixed page size,
1242 * PTE references were taken both at allocation time (to make
1243 * the GPU see the region as sparse), and when mapping memory
1246 * The latter was handled above, and the remaining references
1247 * are dealt with here.
1249 nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1252 /* Remove VMA from the list of allocated nodes. */
1253 rb_erase(&vma->tree, &vmm->root);
1255 /* Merge VMA back into the free list. */
1256 vma->page = NVKM_VMA_PAGE_NONE;
1257 vma->refd = NVKM_VMA_PAGE_NONE;
1260 nvkm_vmm_put_region(vmm, vma);
1264 nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1266 struct nvkm_vma *vma = *pvma;
1268 mutex_lock(&vmm->mutex);
1269 nvkm_vmm_put_locked(vmm, vma);
1270 mutex_unlock(&vmm->mutex);
1276 nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1277 u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1279 const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1280 struct rb_node *node = NULL, *temp;
1281 struct nvkm_vma *vma = NULL, *tmp;
1285 VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1286 "shift: %d align: %d size: %016llx",
1287 getref, mapref, sparse, shift, align, size);
1289 /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1290 if (unlikely(!size || (!getref && !mapref && sparse))) {
1291 VMM_DEBUG(vmm, "args %016llx %d %d %d",
1292 size, getref, mapref, sparse);
1296 /* Tesla-class GPUs can only select page size per-PDE, which means
1297 * we're required to know the mapping granularity up-front to find
1298 * a suitable region of address-space.
1300 * The same goes if we're requesting up-front allocation of PTES.
1302 if (unlikely((getref || vmm->func->page_block) && !shift)) {
1303 VMM_DEBUG(vmm, "page size required: %d %016llx",
1304 getref, vmm->func->page_block);
1308 /* If a specific page size was requested, determine its index and
1309 * make sure the requested size is a multiple of the page size.
1312 for (page = vmm->func->page; page->shift; page++) {
1313 if (shift == page->shift)
1317 if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1318 VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1321 align = max_t(u8, align, shift);
1323 align = max_t(u8, align, 12);
1326 /* Locate smallest block that can possibly satisfy the allocation. */
1327 temp = vmm->free.rb_node;
1329 struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1330 if (this->size < size) {
1331 temp = temp->rb_right;
1334 temp = temp->rb_left;
1338 if (unlikely(!node))
1341 /* Take into account alignment restrictions, trying larger blocks
1342 * in turn until we find a suitable free block.
1345 struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1346 struct nvkm_vma *prev = node(this, prev);
1347 struct nvkm_vma *next = node(this, next);
1348 const int p = page - vmm->func->page;
1351 if (vmm->func->page_block && prev && prev->page != p)
1352 addr = ALIGN(addr, vmm->func->page_block);
1353 addr = ALIGN(addr, 1ULL << align);
1355 tail = this->addr + this->size;
1356 if (vmm->func->page_block && next && next->page != p)
1357 tail = ALIGN_DOWN(tail, vmm->func->page_block);
1359 if (addr <= tail && tail - addr >= size) {
1360 rb_erase(&this->tree, &vmm->free);
1364 } while ((node = rb_next(node)));
1369 /* If the VMA we found isn't already exactly the requested size,
1370 * it needs to be split, and the remaining free blocks returned.
1372 if (addr != vma->addr) {
1373 if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1374 nvkm_vmm_put_region(vmm, vma);
1377 nvkm_vmm_free_insert(vmm, vma);
1381 if (size != vma->size) {
1382 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1383 nvkm_vmm_put_region(vmm, vma);
1386 nvkm_vmm_free_insert(vmm, tmp);
1389 /* Pre-allocate page tables and/or setup sparse mappings. */
1390 if (sparse && getref)
1391 ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1393 ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1395 ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1399 nvkm_vmm_put_region(vmm, vma);
1403 vma->mapref = mapref && !getref;
1404 vma->sparse = sparse;
1405 vma->page = page - vmm->func->page;
1406 vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1408 nvkm_vmm_node_insert(vmm, vma);
1414 nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1417 mutex_lock(&vmm->mutex);
1418 ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1419 mutex_unlock(&vmm->mutex);
1424 nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1426 if (inst && vmm && vmm->func->part) {
1427 mutex_lock(&vmm->mutex);
1428 vmm->func->part(vmm, inst);
1429 mutex_unlock(&vmm->mutex);
1434 nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1437 if (vmm->func->join) {
1438 mutex_lock(&vmm->mutex);
1439 ret = vmm->func->join(vmm, inst);
1440 mutex_unlock(&vmm->mutex);
1446 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
1448 const struct nvkm_vmm_desc *desc = it->desc;
1449 const int type = desc->type == SPT;
1450 nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1455 nvkm_vmm_boot(struct nvkm_vmm *vmm)
1457 const struct nvkm_vmm_page *page = vmm->func->page;
1458 const u64 limit = vmm->limit - vmm->start;
1461 while (page[1].shift)
1464 ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1468 nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false,
1469 nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1470 vmm->bootstrapped = true;
1475 nvkm_vmm_del(struct kref *kref)
1477 struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1483 nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1485 struct nvkm_vmm *vmm = *pvmm;
1487 kref_put(&vmm->kref, nvkm_vmm_del);
1493 nvkm_vmm_ref(struct nvkm_vmm *vmm)
1496 kref_get(&vmm->kref);
1501 nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1502 u32 argc, struct lock_class_key *key, const char *name,
1503 struct nvkm_vmm **pvmm)
1505 struct nvkm_mmu *mmu = device->mmu;
1506 struct nvkm_vmm *vmm = NULL;
1508 ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, key, name, &vmm);
1510 nvkm_vmm_unref(&vmm);