drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/dma-fence-array.h>
  29 #include <linux/interval_tree_generic.h>
  30 #include <linux/idr.h>
  31
  32 #include <drm/amdgpu_drm.h>
  33 #include "amdgpu.h"
  34 #include "amdgpu_trace.h"
  35 #include "amdgpu_amdkfd.h"
  36 #include "amdgpu_gmc.h"
  37 #include "amdgpu_xgmi.h"
  38
  39 /**
  40  * DOC: GPUVM
  41  *
  42  * GPUVM is similar to the legacy gart on older asics, however
  43  * rather than there being a single global gart table
  44  * for the entire GPU, there are multiple VM page tables active
  45  * at any given time.  The VM page tables can contain a mix
  46  * vram pages and system memory pages and system memory pages
  47  * can be mapped as snooped (cached system pages) or unsnooped
  48  * (uncached system pages).
  49  * Each VM has an ID associated with it and there is a page table
  50  * associated with each VMID.  When execting a command buffer,
  51  * the kernel tells the the ring what VMID to use for that command
  52  * buffer.  VMIDs are allocated dynamically as commands are submitted.
  53  * The userspace drivers maintain their own address space and the kernel
  54  * sets up their pages tables accordingly when they submit their
  55  * command buffers and a VMID is assigned.
  56  * Cayman/Trinity support up to 8 active VMs at any given time;
  57  * SI supports 16.
  58  */
  59
  60 #define START(node) ((node)->start)
  61 #define LAST(node) ((node)->last)
  62
  63 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  64                      START, LAST, static, amdgpu_vm_it)
  65
  66 #undef START
  67 #undef LAST
  68
  69 /**
  70  * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
  71  */
  72 struct amdgpu_prt_cb {
  73
  74         /**
  75          * @adev: amdgpu device
  76          */
  77         struct amdgpu_device *adev;
  78
  79         /**
  80          * @cb: callback
  81          */
  82         struct dma_fence_cb cb;
  83 };
  84
  85 /**
  86  * amdgpu_vm_level_shift - return the addr shift for each level
  87  *
  88  * @adev: amdgpu_device pointer
  89  * @level: VMPT level
  90  *
  91  * Returns:
  92  * The number of bits the pfn needs to be right shifted for a level.
  93  */
  94 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
  95                                       unsigned level)
  96 {
  97         unsigned shift = 0xff;
  98
  99         switch (level) {
 100         case AMDGPU_VM_PDB2:
 101         case AMDGPU_VM_PDB1:
 102         case AMDGPU_VM_PDB0:
 103                 shift = 9 * (AMDGPU_VM_PDB0 - level) +
 104                         adev->vm_manager.block_size;
 105                 break;
 106         case AMDGPU_VM_PTB:
 107                 shift = 0;
 108                 break;
 109         default:
 110                 dev_err(adev->dev, "the level%d isn't supported.\n", level);
 111         }
 112
 113         return shift;
 114 }
 115
 116 /**
 117  * amdgpu_vm_num_entries - return the number of entries in a PD/PT
 118  *
 119  * @adev: amdgpu_device pointer
 120  * @level: VMPT level
 121  *
 122  * Returns:
 123  * The number of entries in a page directory or page table.
 124  */
 125 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
 126                                       unsigned level)
 127 {
 128         unsigned shift = amdgpu_vm_level_shift(adev,
 129                                                adev->vm_manager.root_level);
 130
 131         if (level == adev->vm_manager.root_level)
 132                 /* For the root directory */
 133                 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift;
 134         else if (level != AMDGPU_VM_PTB)
 135                 /* Everything in between */
 136                 return 512;
 137         else
 138                 /* For the page tables on the leaves */
 139                 return AMDGPU_VM_PTE_COUNT(adev);
 140 }
 141
 142 /**
 143  * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD
 144  *
 145  * @adev: amdgpu_device pointer
 146  *
 147  * Returns:
 148  * The number of entries in the root page directory which needs the ATS setting.
 149  */
 150 static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev)
 151 {
 152         unsigned shift;
 153
 154         shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level);
 155         return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT);
 156 }
 157
 158 /**
 159  * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT
 160  *
 161  * @adev: amdgpu_device pointer
 162  * @level: VMPT level
 163  *
 164  * Returns:
 165  * The mask to extract the entry number of a PD/PT from an address.
 166  */
 167 static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev,
 168                                        unsigned int level)
 169 {
 170         if (level <= adev->vm_manager.root_level)
 171                 return 0xffffffff;
 172         else if (level != AMDGPU_VM_PTB)
 173                 return 0x1ff;
 174         else
 175                 return AMDGPU_VM_PTE_COUNT(adev) - 1;
 176 }
 177
 178 /**
 179  * amdgpu_vm_bo_size - returns the size of the BOs in bytes
 180  *
 181  * @adev: amdgpu_device pointer
 182  * @level: VMPT level
 183  *
 184  * Returns:
 185  * The size of the BO for a page directory or page table in bytes.
 186  */
 187 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
 188 {
 189         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
 190 }
 191
 192 /**
 193  * amdgpu_vm_bo_evicted - vm_bo is evicted
 194  *
 195  * @vm_bo: vm_bo which is evicted
 196  *
 197  * State for PDs/PTs and per VM BOs which are not at the location they should
 198  * be.
 199  */
 200 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 201 {
 202         struct amdgpu_vm *vm = vm_bo->vm;
 203         struct amdgpu_bo *bo = vm_bo->bo;
 204
 205         vm_bo->moved = true;
 206         if (bo->tbo.type == ttm_bo_type_kernel)
 207                 list_move(&vm_bo->vm_status, &vm->evicted);
 208         else
 209                 list_move_tail(&vm_bo->vm_status, &vm->evicted);
 210 }
 211
 212 /**
 213  * amdgpu_vm_bo_relocated - vm_bo is reloacted
 214  *
 215  * @vm_bo: vm_bo which is relocated
 216  *
 217  * State for PDs/PTs which needs to update their parent PD.
 218  */
 219 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 220 {
 221         list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
 222 }
 223
 224 /**
 225  * amdgpu_vm_bo_moved - vm_bo is moved
 226  *
 227  * @vm_bo: vm_bo which is moved
 228  *
 229  * State for per VM BOs which are moved, but that change is not yet reflected
 230  * in the page tables.
 231  */
 232 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 233 {
 234         list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
 235 }
 236
 237 /**
 238  * amdgpu_vm_bo_idle - vm_bo is idle
 239  *
 240  * @vm_bo: vm_bo which is now idle
 241  *
 242  * State for PDs/PTs and per VM BOs which have gone through the state machine
 243  * and are now idle.
 244  */
 245 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 246 {
 247         list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
 248         vm_bo->moved = false;
 249 }
 250
 251 /**
 252  * amdgpu_vm_bo_invalidated - vm_bo is invalidated
 253  *
 254  * @vm_bo: vm_bo which is now invalidated
 255  *
 256  * State for normal BOs which are invalidated and that change not yet reflected
 257  * in the PTs.
 258  */
 259 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
 260 {
 261         spin_lock(&vm_bo->vm->invalidated_lock);
 262         list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
 263         spin_unlock(&vm_bo->vm->invalidated_lock);
 264 }
 265
 266 /**
 267  * amdgpu_vm_bo_done - vm_bo is done
 268  *
 269  * @vm_bo: vm_bo which is now done
 270  *
 271  * State for normal BOs which are invalidated and that change has been updated
 272  * in the PTs.
 273  */
 274 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
 275 {
 276         spin_lock(&vm_bo->vm->invalidated_lock);
 277         list_del_init(&vm_bo->vm_status);
 278         spin_unlock(&vm_bo->vm->invalidated_lock);
 279 }
 280
 281 /**
 282  * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
 283  *
 284  * @base: base structure for tracking BO usage in a VM
 285  * @vm: vm to which bo is to be added
 286  * @bo: amdgpu buffer object
 287  *
 288  * Initialize a bo_va_base structure and add it to the appropriate lists
 289  *
 290  */
 291 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 292                                    struct amdgpu_vm *vm,
 293                                    struct amdgpu_bo *bo)
 294 {
 295         base->vm = vm;
 296         base->bo = bo;
 297         base->next = NULL;
 298         INIT_LIST_HEAD(&base->vm_status);
 299
 300         if (!bo)
 301                 return;
 302         base->next = bo->vm_bo;
 303         bo->vm_bo = base;
 304
 305         if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
 306                 return;
 307
 308         vm->bulk_moveable = false;
 309         if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
 310                 amdgpu_vm_bo_relocated(base);
 311         else
 312                 amdgpu_vm_bo_idle(base);
 313
 314         if (bo->preferred_domains &
 315             amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
 316                 return;
 317
 318         /*
 319          * we checked all the prerequisites, but it looks like this per vm bo
 320          * is currently evicted. add the bo to the evicted list to make sure it
 321          * is validated on next vm use to avoid fault.
 322          * */
 323         amdgpu_vm_bo_evicted(base);
 324 }
 325
 326 /**
 327  * amdgpu_vm_pt_parent - get the parent page directory
 328  *
 329  * @pt: child page table
 330  *
 331  * Helper to get the parent entry for the child page table. NULL if we are at
 332  * the root page directory.
 333  */
 334 static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt)
 335 {
 336         struct amdgpu_bo *parent = pt->base.bo->parent;
 337
 338         if (!parent)
 339                 return NULL;
 340
 341         return container_of(parent->vm_bo, struct amdgpu_vm_pt, base);
 342 }
 343
 344 /**
 345  * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
 346  */
 347 struct amdgpu_vm_pt_cursor {
 348         uint64_t pfn;
 349         struct amdgpu_vm_pt *parent;
 350         struct amdgpu_vm_pt *entry;
 351         unsigned level;
 352 };
 353
 354 /**
 355  * amdgpu_vm_pt_start - start PD/PT walk
 356  *
 357  * @adev: amdgpu_device pointer
 358  * @vm: amdgpu_vm structure
 359  * @start: start address of the walk
 360  * @cursor: state to initialize
 361  *
 362  * Initialize a amdgpu_vm_pt_cursor to start a walk.
 363  */
 364 static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
 365                                struct amdgpu_vm *vm, uint64_t start,
 366                                struct amdgpu_vm_pt_cursor *cursor)
 367 {
 368         cursor->pfn = start;
 369         cursor->parent = NULL;
 370         cursor->entry = &vm->root;
 371         cursor->level = adev->vm_manager.root_level;
 372 }
 373
 374 /**
 375  * amdgpu_vm_pt_descendant - go to child node
 376  *
 377  * @adev: amdgpu_device pointer
 378  * @cursor: current state
 379  *
 380  * Walk to the child node of the current node.
 381  * Returns:
 382  * True if the walk was possible, false otherwise.
 383  */
 384 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
 385                                     struct amdgpu_vm_pt_cursor *cursor)
 386 {
 387         unsigned mask, shift, idx;
 388
 389         if (!cursor->entry->entries)
 390                 return false;
 391
 392         BUG_ON(!cursor->entry->base.bo);
 393         mask = amdgpu_vm_entries_mask(adev, cursor->level);
 394         shift = amdgpu_vm_level_shift(adev, cursor->level);
 395
 396         ++cursor->level;
 397         idx = (cursor->pfn >> shift) & mask;
 398         cursor->parent = cursor->entry;
 399         cursor->entry = &cursor->entry->entries[idx];
 400         return true;
 401 }
 402
 403 /**
 404  * amdgpu_vm_pt_sibling - go to sibling node
 405  *
 406  * @adev: amdgpu_device pointer
 407  * @cursor: current state
 408  *
 409  * Walk to the sibling node of the current node.
 410  * Returns:
 411  * True if the walk was possible, false otherwise.
 412  */
 413 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
 414                                  struct amdgpu_vm_pt_cursor *cursor)
 415 {
 416         unsigned shift, num_entries;
 417
 418         /* Root doesn't have a sibling */
 419         if (!cursor->parent)
 420                 return false;
 421
 422         /* Go to our parents and see if we got a sibling */
 423         shift = amdgpu_vm_level_shift(adev, cursor->level - 1);
 424         num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);
 425
 426         if (cursor->entry == &cursor->parent->entries[num_entries - 1])
 427                 return false;
 428
 429         cursor->pfn += 1ULL << shift;
 430         cursor->pfn &= ~((1ULL << shift) - 1);
 431         ++cursor->entry;
 432         return true;
 433 }
 434
 435 /**
 436  * amdgpu_vm_pt_ancestor - go to parent node
 437  *
 438  * @cursor: current state
 439  *
 440  * Walk to the parent node of the current node.
 441  * Returns:
 442  * True if the walk was possible, false otherwise.
 443  */
 444 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
 445 {
 446         if (!cursor->parent)
 447                 return false;
 448
 449         --cursor->level;
 450         cursor->entry = cursor->parent;
 451         cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
 452         return true;
 453 }
 454
 455 /**
 456  * amdgpu_vm_pt_next - get next PD/PT in hieratchy
 457  *
 458  * @adev: amdgpu_device pointer
 459  * @cursor: current state
 460  *
 461  * Walk the PD/PT tree to the next node.
 462  */
 463 static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
 464                               struct amdgpu_vm_pt_cursor *cursor)
 465 {
 466         /* First try a newborn child */
 467         if (amdgpu_vm_pt_descendant(adev, cursor))
 468                 return;
 469
 470         /* If that didn't worked try to find a sibling */
 471         while (!amdgpu_vm_pt_sibling(adev, cursor)) {
 472                 /* No sibling, go to our parents and grandparents */
 473                 if (!amdgpu_vm_pt_ancestor(cursor)) {
 474                         cursor->pfn = ~0ll;
 475                         return;
 476                 }
 477         }
 478 }
 479
 480 /**
 481  * amdgpu_vm_pt_first_dfs - start a deep first search
 482  *
 483  * @adev: amdgpu_device structure
 484  * @vm: amdgpu_vm structure
 485  * @cursor: state to initialize
 486  *
 487  * Starts a deep first traversal of the PD/PT tree.
 488  */
 489 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
 490                                    struct amdgpu_vm *vm,
 491                                    struct amdgpu_vm_pt_cursor *start,
 492                                    struct amdgpu_vm_pt_cursor *cursor)
 493 {
 494         if (start)
 495                 *cursor = *start;
 496         else
 497                 amdgpu_vm_pt_start(adev, vm, 0, cursor);
 498         while (amdgpu_vm_pt_descendant(adev, cursor));
 499 }
 500
 501 /**
 502  * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
 503  *
 504  * @start: starting point for the search
 505  * @entry: current entry
 506  *
 507  * Returns:
 508  * True when the search should continue, false otherwise.
 509  */
 510 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
 511                                       struct amdgpu_vm_pt *entry)
 512 {
 513         return entry && (!start || entry != start->entry);
 514 }
 515
 516 /**
 517  * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
 518  *
 519  * @adev: amdgpu_device structure
 520  * @cursor: current state
 521  *
 522  * Move the cursor to the next node in a deep first search.
 523  */
 524 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
 525                                   struct amdgpu_vm_pt_cursor *cursor)
 526 {
 527         if (!cursor->entry)
 528                 return;
 529
 530         if (!cursor->parent)
 531                 cursor->entry = NULL;
 532         else if (amdgpu_vm_pt_sibling(adev, cursor))
 533                 while (amdgpu_vm_pt_descendant(adev, cursor));
 534         else
 535                 amdgpu_vm_pt_ancestor(cursor);
 536 }
 537
 538 /**
 539  * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
 540  */
 541 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)          \
 542         for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)),          \
 543              (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
 544              amdgpu_vm_pt_continue_dfs((start), (entry));                       \
 545              (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
 546
 547 /**
 548  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
 549  *
 550  * @vm: vm providing the BOs
 551  * @validated: head of validation list
 552  * @entry: entry to add
 553  *
 554  * Add the page directory to the list of BOs to
 555  * validate for command submission.
 556  */
 557 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 558                          struct list_head *validated,
 559                          struct amdgpu_bo_list_entry *entry)
 560 {
 561         entry->priority = 0;
 562         entry->tv.bo = &vm->root.base.bo->tbo;
 563         /* One for the VM updates, one for TTM and one for the CS job */
 564         entry->tv.num_shared = 3;
 565         entry->user_pages = NULL;
 566         list_add(&entry->tv.head, validated);
 567 }
 568
 569 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo)
 570 {
 571         struct amdgpu_bo *abo;
 572         struct amdgpu_vm_bo_base *bo_base;
 573
 574         if (!amdgpu_bo_is_amdgpu_bo(bo))
 575                 return;
 576
 577         if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT)
 578                 return;
 579
 580         abo = ttm_to_amdgpu_bo(bo);
 581         if (!abo->parent)
 582                 return;
 583         for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) {
 584                 struct amdgpu_vm *vm = bo_base->vm;
 585
 586                 if (abo->tbo.resv == vm->root.base.bo->tbo.resv)
 587                         vm->bulk_moveable = false;
 588         }
 589
 590 }
 591 /**
 592  * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
 593  *
 594  * @adev: amdgpu device pointer
 595  * @vm: vm providing the BOs
 596  *
 597  * Move all BOs to the end of LRU and remember their positions to put them
 598  * together.
 599  */
 600 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 601                                 struct amdgpu_vm *vm)
 602 {
 603         struct ttm_bo_global *glob = adev->mman.bdev.glob;
 604         struct amdgpu_vm_bo_base *bo_base;
 605
 606 #if 0
 607         if (vm->bulk_moveable) {
 608                 spin_lock(&glob->lru_lock);
 609                 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);
 610                 spin_unlock(&glob->lru_lock);
 611                 return;
 612         }
 613 #endif
 614
 615         memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));
 616
 617         spin_lock(&glob->lru_lock);
 618         list_for_each_entry(bo_base, &vm->idle, vm_status) {
 619                 struct amdgpu_bo *bo = bo_base->bo;
 620
 621                 if (!bo->parent)
 622                         continue;
 623
 624                 ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move);
 625                 if (bo->shadow)
 626                         ttm_bo_move_to_lru_tail(&bo->shadow->tbo,
 627                                                 &vm->lru_bulk_move);
 628         }
 629         spin_unlock(&glob->lru_lock);
 630
 631         vm->bulk_moveable = true;
 632 }
 633
 634 /**
 635  * amdgpu_vm_validate_pt_bos - validate the page table BOs
 636  *
 637  * @adev: amdgpu device pointer
 638  * @vm: vm providing the BOs
 639  * @validate: callback to do the validation
 640  * @param: parameter for the validation callback
 641  *
 642  * Validate the page table BOs on command submission if neccessary.
 643  *
 644  * Returns:
 645  * Validation result.
 646  */
 647 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 648                               int (*validate)(void *p, struct amdgpu_bo *bo),
 649                               void *param)
 650 {
 651         struct amdgpu_vm_bo_base *bo_base, *tmp;
 652         int r = 0;
 653
 654         vm->bulk_moveable &= list_empty(&vm->evicted);
 655
 656         list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
 657                 struct amdgpu_bo *bo = bo_base->bo;
 658
 659                 r = validate(param, bo);
 660                 if (r)
 661                         break;
 662
 663                 if (bo->tbo.type != ttm_bo_type_kernel) {
 664                         amdgpu_vm_bo_moved(bo_base);
 665                 } else {
 666                         vm->update_funcs->map_table(bo);
 667                         if (bo->parent)
 668                                 amdgpu_vm_bo_relocated(bo_base);
 669                         else
 670                                 amdgpu_vm_bo_idle(bo_base);
 671                 }
 672         }
 673
 674         return r;
 675 }
 676
 677 /**
 678  * amdgpu_vm_ready - check VM is ready for updates
 679  *
 680  * @vm: VM to check
 681  *
 682  * Check if all VM PDs/PTs are ready for updates
 683  *
 684  * Returns:
 685  * True if eviction list is empty.
 686  */
 687 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 688 {
 689         return list_empty(&vm->evicted);
 690 }
 691
 692 /**
 693  * amdgpu_vm_clear_bo - initially clear the PDs/PTs
 694  *
 695  * @adev: amdgpu_device pointer
 696  * @vm: VM to clear BO from
 697  * @bo: BO to clear
 698  *
 699  * Root PD needs to be reserved when calling this.
 700  *
 701  * Returns:
 702  * 0 on success, errno otherwise.
 703  */
 704 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 705                               struct amdgpu_vm *vm,
 706                               struct amdgpu_bo *bo)
 707 {
 708         struct ttm_operation_ctx ctx = { true, false };
 709         unsigned level = adev->vm_manager.root_level;
 710         struct amdgpu_vm_update_params params;
 711         struct amdgpu_bo *ancestor = bo;
 712         unsigned entries, ats_entries;
 713         uint64_t addr;
 714         int r;
 715
 716         /* Figure out our place in the hierarchy */
 717         if (ancestor->parent) {
 718                 ++level;
 719                 while (ancestor->parent->parent) {
 720                         ++level;
 721                         ancestor = ancestor->parent;
 722                 }
 723         }
 724
 725         entries = amdgpu_bo_size(bo) / 8;
 726         if (!vm->pte_support_ats) {
 727                 ats_entries = 0;
 728
 729         } else if (!bo->parent) {
 730                 ats_entries = amdgpu_vm_num_ats_entries(adev);
 731                 ats_entries = min(ats_entries, entries);
 732                 entries -= ats_entries;
 733
 734         } else {
 735                 struct amdgpu_vm_pt *pt;
 736
 737                 pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base);
 738                 ats_entries = amdgpu_vm_num_ats_entries(adev);
 739                 if ((pt - vm->root.entries) >= ats_entries) {
 740                         ats_entries = 0;
 741                 } else {
 742                         ats_entries = entries;
 743                         entries = 0;
 744                 }
 745         }
 746
 747         r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 748         if (r)
 749                 return r;
 750
 751         if (bo->shadow) {
 752                 r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement,
 753                                     &ctx);
 754                 if (r)
 755                         return r;
 756         }
 757
 758         r = vm->update_funcs->map_table(bo);
 759         if (r)
 760                 return r;
 761
 762         memset(&params, 0, sizeof(params));
 763         params.adev = adev;
 764         params.vm = vm;
 765
 766         r = vm->update_funcs->prepare(&params, AMDGPU_FENCE_OWNER_KFD, NULL);
 767         if (r)
 768                 return r;
 769
 770         addr = 0;
 771         if (ats_entries) {
 772                 uint64_t value = 0, flags;
 773
 774                 flags = AMDGPU_PTE_DEFAULT_ATC;
 775                 if (level != AMDGPU_VM_PTB) {
 776                         /* Handle leaf PDEs as PTEs */
 777                         flags |= AMDGPU_PDE_PTE;
 778                         amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
 779                 }
 780
 781                 r = vm->update_funcs->update(&params, bo, addr, 0, ats_entries,
 782                                              value, flags);
 783                 if (r)
 784                         return r;
 785
 786                 addr += ats_entries * 8;
 787         }
 788
 789         if (entries) {
 790                 uint64_t value = 0, flags = 0;
 791
 792                 if (adev->asic_type >= CHIP_VEGA10) {
 793                         if (level != AMDGPU_VM_PTB) {
 794                                 /* Handle leaf PDEs as PTEs */
 795                                 flags |= AMDGPU_PDE_PTE;
 796                                 amdgpu_gmc_get_vm_pde(adev, level,
 797                                                       &value, &flags);
 798                         } else {
 799                                 /* Workaround for fault priority problem on GMC9 */
 800                                 flags = AMDGPU_PTE_EXECUTABLE;
 801                         }
 802                 }
 803
 804                 r = vm->update_funcs->update(&params, bo, addr, 0, entries,
 805                                              value, flags);
 806                 if (r)
 807                         return r;
 808         }
 809
 810         return vm->update_funcs->commit(&params, NULL);
 811 }
 812
 813 /**
 814  * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation
 815  *
 816  * @adev: amdgpu_device pointer
 817  * @vm: requesting vm
 818  * @bp: resulting BO allocation parameters
 819  */
 820 static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 821                                int level, struct amdgpu_bo_param *bp)
 822 {
 823         memset(bp, 0, sizeof(*bp));
 824
 825         bp->size = amdgpu_vm_bo_size(adev, level);
 826         bp->byte_align = AMDGPU_GPU_PAGE_SIZE;
 827         bp->domain = AMDGPU_GEM_DOMAIN_VRAM;
 828         bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain);
 829         bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
 830                 AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 831         if (vm->use_cpu_for_update)
 832                 bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 833         else if (!vm->root.base.bo || vm->root.base.bo->shadow)
 834                 bp->flags |= AMDGPU_GEM_CREATE_SHADOW;
 835         bp->type = ttm_bo_type_kernel;
 836         if (vm->root.base.bo)
 837                 bp->resv = vm->root.base.bo->tbo.resv;
 838 }
 839
 840 /**
 841  * amdgpu_vm_alloc_pts - Allocate a specific page table
 842  *
 843  * @adev: amdgpu_device pointer
 844  * @vm: VM to allocate page tables for
 845  * @cursor: Which page table to allocate
 846  *
 847  * Make sure a specific page table or directory is allocated.
 848  *
 849  * Returns:
 850  * 1 if page table needed to be allocated, 0 if page table was already
 851  * allocated, negative errno if an error occurred.
 852  */
 853 static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 854                                struct amdgpu_vm *vm,
 855                                struct amdgpu_vm_pt_cursor *cursor)
 856 {
 857         struct amdgpu_vm_pt *entry = cursor->entry;
 858         struct amdgpu_bo_param bp;
 859         struct amdgpu_bo *pt;
 860         int r;
 861
 862         if (cursor->level < AMDGPU_VM_PTB && !entry->entries) {
 863                 unsigned num_entries;
 864
 865                 num_entries = amdgpu_vm_num_entries(adev, cursor->level);
 866                 entry->entries = kvmalloc_array(num_entries,
 867                                                 sizeof(*entry->entries),
 868                                                 GFP_KERNEL | __GFP_ZERO);
 869                 if (!entry->entries)
 870                         return -ENOMEM;
 871         }
 872
 873         if (entry->base.bo)
 874                 return 0;
 875
 876         amdgpu_vm_bo_param(adev, vm, cursor->level, &bp);
 877
 878         r = amdgpu_bo_create(adev, &bp, &pt);
 879         if (r)
 880                 return r;
 881
 882         /* Keep a reference to the root directory to avoid
 883          * freeing them up in the wrong order.
 884          */
 885         pt->parent = amdgpu_bo_ref(cursor->parent->base.bo);
 886         amdgpu_vm_bo_base_init(&entry->base, vm, pt);
 887
 888         r = amdgpu_vm_clear_bo(adev, vm, pt);
 889         if (r)
 890                 goto error_free_pt;
 891
 892         return 0;
 893
 894 error_free_pt:
 895         amdgpu_bo_unref(&pt->shadow);
 896         amdgpu_bo_unref(&pt);
 897         return r;
 898 }
 899
 900 /**
 901  * amdgpu_vm_free_table - fre one PD/PT
 902  *
 903  * @entry: PDE to free
 904  */
 905 static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry)
 906 {
 907         if (entry->base.bo) {
 908                 entry->base.bo->vm_bo = NULL;
 909                 list_del(&entry->base.vm_status);
 910                 amdgpu_bo_unref(&entry->base.bo->shadow);
 911                 amdgpu_bo_unref(&entry->base.bo);
 912         }
 913         kvfree(entry->entries);
 914         entry->entries = NULL;
 915 }
 916
 917 /**
 918  * amdgpu_vm_free_pts - free PD/PT levels
 919  *
 920  * @adev: amdgpu device structure
 921  * @vm: amdgpu vm structure
 922  * @start: optional cursor where to start freeing PDs/PTs
 923  *
 924  * Free the page directory or page table level and all sub levels.
 925  */
 926 static void amdgpu_vm_free_pts(struct amdgpu_device *adev,
 927                                struct amdgpu_vm *vm,
 928                                struct amdgpu_vm_pt_cursor *start)
 929 {
 930         struct amdgpu_vm_pt_cursor cursor;
 931         struct amdgpu_vm_pt *entry;
 932
 933         vm->bulk_moveable = false;
 934
 935         for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
 936                 amdgpu_vm_free_table(entry);
 937
 938         if (start)
 939                 amdgpu_vm_free_table(start->entry);
 940 }
 941
 942 /**
 943  * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
 944  *
 945  * @adev: amdgpu_device pointer
 946  */
 947 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
 948 {
 949         const struct amdgpu_ip_block *ip_block;
 950         bool has_compute_vm_bug;
 951         struct amdgpu_ring *ring;
 952         int i;
 953
 954         has_compute_vm_bug = false;
 955
 956         ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 957         if (ip_block) {
 958                 /* Compute has a VM bug for GFX version < 7.
 959                    Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
 960                 if (ip_block->version->major <= 7)
 961                         has_compute_vm_bug = true;
 962                 else if (ip_block->version->major == 8)
 963                         if (adev->gfx.mec_fw_version < 673)
 964                                 has_compute_vm_bug = true;
 965         }
 966
 967         for (i = 0; i < adev->num_rings; i++) {
 968                 ring = adev->rings[i];
 969                 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
 970                         /* only compute rings */
 971                         ring->has_compute_vm_bug = has_compute_vm_bug;
 972                 else
 973                         ring->has_compute_vm_bug = false;
 974         }
 975 }
 976
 977 /**
 978  * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
 979  *
 980  * @ring: ring on which the job will be submitted
 981  * @job: job to submit
 982  *
 983  * Returns:
 984  * True if sync is needed.
 985  */
 986 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 987                                   struct amdgpu_job *job)
 988 {
 989         struct amdgpu_device *adev = ring->adev;
 990         unsigned vmhub = ring->funcs->vmhub;
 991         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 992         struct amdgpu_vmid *id;
 993         bool gds_switch_needed;
 994         bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
 995
 996         if (job->vmid == 0)
 997                 return false;
 998         id = &id_mgr->ids[job->vmid];
 999         gds_switch_needed = ring->funcs->emit_gds_switch && (
1000                 id->gds_base != job->gds_base ||
1001                 id->gds_size != job->gds_size ||
1002                 id->gws_base != job->gws_base ||
1003                 id->gws_size != job->gws_size ||
1004                 id->oa_base != job->oa_base ||
1005                 id->oa_size != job->oa_size);
1006
1007         if (amdgpu_vmid_had_gpu_reset(adev, id))
1008                 return true;
1009
1010         return vm_flush_needed || gds_switch_needed;
1011 }
1012
1013 /**
1014  * amdgpu_vm_flush - hardware flush the vm
1015  *
1016  * @ring: ring to use for flush
1017  * @job:  related job
1018  * @need_pipe_sync: is pipe sync needed
1019  *
1020  * Emit a VM flush when it is necessary.
1021  *
1022  * Returns:
1023  * 0 on success, errno otherwise.
1024  */
1025 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
1026 {
1027         struct amdgpu_device *adev = ring->adev;
1028         unsigned vmhub = ring->funcs->vmhub;
1029         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
1030         struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
1031         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
1032                 id->gds_base != job->gds_base ||
1033                 id->gds_size != job->gds_size ||
1034                 id->gws_base != job->gws_base ||
1035                 id->gws_size != job->gws_size ||
1036                 id->oa_base != job->oa_base ||
1037                 id->oa_size != job->oa_size);
1038         bool vm_flush_needed = job->vm_needs_flush;
1039         bool pasid_mapping_needed = id->pasid != job->pasid ||
1040                 !id->pasid_mapping ||
1041                 !dma_fence_is_signaled(id->pasid_mapping);
1042         struct dma_fence *fence = NULL;
1043         unsigned patch_offset = 0;
1044         int r;
1045
1046         if (amdgpu_vmid_had_gpu_reset(adev, id)) {
1047                 gds_switch_needed = true;
1048                 vm_flush_needed = true;
1049                 pasid_mapping_needed = true;
1050         }
1051
1052         gds_switch_needed &= !!ring->funcs->emit_gds_switch;
1053         vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
1054                         job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
1055         pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
1056                 ring->funcs->emit_wreg;
1057
1058         if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
1059                 return 0;
1060
1061         if (ring->funcs->init_cond_exec)
1062                 patch_offset = amdgpu_ring_init_cond_exec(ring);
1063
1064         if (need_pipe_sync)
1065                 amdgpu_ring_emit_pipeline_sync(ring);
1066
1067         if (vm_flush_needed) {
1068                 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
1069                 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
1070         }
1071
1072         if (pasid_mapping_needed)
1073                 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
1074
1075         if (vm_flush_needed || pasid_mapping_needed) {
1076                 r = amdgpu_fence_emit(ring, &fence, 0);
1077                 if (r)
1078                         return r;
1079         }
1080
1081         if (vm_flush_needed) {
1082                 mutex_lock(&id_mgr->lock);
1083                 dma_fence_put(id->last_flush);
1084                 id->last_flush = dma_fence_get(fence);
1085                 id->current_gpu_reset_count =
1086                         atomic_read(&adev->gpu_reset_counter);
1087                 mutex_unlock(&id_mgr->lock);
1088         }
1089
1090         if (pasid_mapping_needed) {
1091                 id->pasid = job->pasid;
1092                 dma_fence_put(id->pasid_mapping);
1093                 id->pasid_mapping = dma_fence_get(fence);
1094         }
1095         dma_fence_put(fence);
1096
1097         if (ring->funcs->emit_gds_switch && gds_switch_needed) {
1098                 id->gds_base = job->gds_base;
1099                 id->gds_size = job->gds_size;
1100                 id->gws_base = job->gws_base;
1101                 id->gws_size = job->gws_size;
1102                 id->oa_base = job->oa_base;
1103                 id->oa_size = job->oa_size;
1104                 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
1105                                             job->gds_size, job->gws_base,
1106                                             job->gws_size, job->oa_base,
1107                                             job->oa_size);
1108         }
1109
1110         if (ring->funcs->patch_cond_exec)
1111                 amdgpu_ring_patch_cond_exec(ring, patch_offset);
1112
1113         /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
1114         if (ring->funcs->emit_switch_buffer) {
1115                 amdgpu_ring_emit_switch_buffer(ring);
1116                 amdgpu_ring_emit_switch_buffer(ring);
1117         }
1118         return 0;
1119 }
1120
1121 /**
1122  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
1123  *
1124  * @vm: requested vm
1125  * @bo: requested buffer object
1126  *
1127  * Find @bo inside the requested vm.
1128  * Search inside the @bos vm list for the requested vm
1129  * Returns the found bo_va or NULL if none is found
1130  *
1131  * Object has to be reserved!
1132  *
1133  * Returns:
1134  * Found bo_va or NULL.
1135  */
1136 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
1137                                        struct amdgpu_bo *bo)
1138 {
1139         struct amdgpu_vm_bo_base *base;
1140
1141         for (base = bo->vm_bo; base; base = base->next) {
1142                 if (base->vm != vm)
1143                         continue;
1144
1145                 return container_of(base, struct amdgpu_bo_va, base);
1146         }
1147         return NULL;
1148 }
1149
1150 /**
1151  * amdgpu_vm_map_gart - Resolve gart mapping of addr
1152  *
1153  * @pages_addr: optional DMA address to use for lookup
1154  * @addr: the unmapped addr
1155  *
1156  * Look up the physical address of the page that the pte resolves
1157  * to.
1158  *
1159  * Returns:
1160  * The pointer for the page table entry.
1161  */
1162 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
1163 {
1164         uint64_t result;
1165
1166         /* page table offset */
1167         result = pages_addr[addr >> PAGE_SHIFT];
1168
1169         /* in case cpu page size != gpu page size*/
1170         result |= addr & (~PAGE_MASK);
1171
1172         result &= 0xFFFFFFFFFFFFF000ULL;
1173
1174         return result;
1175 }
1176
1177 /*
1178  * amdgpu_vm_update_pde - update a single level in the hierarchy
1179  *
1180  * @param: parameters for the update
1181  * @vm: requested vm
1182  * @entry: entry to update
1183  *
1184  * Makes sure the requested entry in parent is up to date.
1185  */
1186 static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params,
1187                                 struct amdgpu_vm *vm,
1188                                 struct amdgpu_vm_pt *entry)
1189 {
1190         struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry);
1191         struct amdgpu_bo *bo = parent->base.bo, *pbo;
1192         uint64_t pde, pt, flags;
1193         unsigned level;
1194
1195         for (level = 0, pbo = bo->parent; pbo; ++level)
1196                 pbo = pbo->parent;
1197
1198         level += params->adev->vm_manager.root_level;
1199         amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
1200         pde = (entry - parent->entries) * 8;
1201         return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags);
1202 }
1203
1204 /*
1205  * amdgpu_vm_invalidate_pds - mark all PDs as invalid
1206  *
1207  * @adev: amdgpu_device pointer
1208  * @vm: related vm
1209  *
1210  * Mark all PD level as invalid after an error.
1211  */
1212 static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev,
1213                                      struct amdgpu_vm *vm)
1214 {
1215         struct amdgpu_vm_pt_cursor cursor;
1216         struct amdgpu_vm_pt *entry;
1217
1218         for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)
1219                 if (entry->base.bo && !entry->base.moved)
1220                         amdgpu_vm_bo_relocated(&entry->base);
1221 }
1222
1223 /*
1224  * amdgpu_vm_update_directories - make sure that all directories are valid
1225  *
1226  * @adev: amdgpu_device pointer
1227  * @vm: requested vm
1228  *
1229  * Makes sure all directories are up to date.
1230  *
1231  * Returns:
1232  * 0 for success, error for failure.
1233  */
1234 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
1235                                  struct amdgpu_vm *vm)
1236 {
1237         struct amdgpu_vm_update_params params;
1238         int r;
1239
1240         if (list_empty(&vm->relocated))
1241                 return 0;
1242
1243         memset(&params, 0, sizeof(params));
1244         params.adev = adev;
1245         params.vm = vm;
1246
1247         r = vm->update_funcs->prepare(&params, AMDGPU_FENCE_OWNER_VM, NULL);
1248         if (r)
1249                 return r;
1250
1251         while (!list_empty(&vm->relocated)) {
1252                 struct amdgpu_vm_pt *entry;
1253
1254                 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt,
1255                                          base.vm_status);
1256                 amdgpu_vm_bo_idle(&entry->base);
1257
1258                 r = amdgpu_vm_update_pde(&params, vm, entry);
1259                 if (r)
1260                         goto error;
1261         }
1262
1263         r = vm->update_funcs->commit(&params, &vm->last_update);
1264         if (r)
1265                 goto error;
1266         return 0;
1267
1268 error:
1269         amdgpu_vm_invalidate_pds(adev, vm);
1270         return r;
1271 }
1272
1273 /**
1274  * amdgpu_vm_update_flags - figure out flags for PTE updates
1275  *
1276  * Make sure to set the right flags for the PTEs at the desired level.
1277  */
1278 static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params,
1279                                    struct amdgpu_bo *bo, unsigned level,
1280                                    uint64_t pe, uint64_t addr,
1281                                    unsigned count, uint32_t incr,
1282                                    uint64_t flags)
1283
1284 {
1285         if (level != AMDGPU_VM_PTB) {
1286                 flags |= AMDGPU_PDE_PTE;
1287                 amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags);
1288
1289         } else if (params->adev->asic_type >= CHIP_VEGA10 &&
1290                    !(flags & AMDGPU_PTE_VALID) &&
1291                    !(flags & AMDGPU_PTE_PRT)) {
1292
1293                 /* Workaround for fault priority problem on GMC9 */
1294                 flags |= AMDGPU_PTE_EXECUTABLE;
1295         }
1296
1297         params->vm->update_funcs->update(params, bo, pe, addr, count, incr,
1298                                          flags);
1299 }
1300
1301 /**
1302  * amdgpu_vm_fragment - get fragment for PTEs
1303  *
1304  * @params: see amdgpu_vm_update_params definition
1305  * @start: first PTE to handle
1306  * @end: last PTE to handle
1307  * @flags: hw mapping flags
1308  * @frag: resulting fragment size
1309  * @frag_end: end of this fragment
1310  *
1311  * Returns the first possible fragment for the start and end address.
1312  */
1313 static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params,
1314                                uint64_t start, uint64_t end, uint64_t flags,
1315                                unsigned int *frag, uint64_t *frag_end)
1316 {
1317         /**
1318          * The MC L1 TLB supports variable sized pages, based on a fragment
1319          * field in the PTE. When this field is set to a non-zero value, page
1320          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1321          * flags are considered valid for all PTEs within the fragment range
1322          * and corresponding mappings are assumed to be physically contiguous.
1323          *
1324          * The L1 TLB can store a single PTE for the whole fragment,
1325          * significantly increasing the space available for translation
1326          * caching. This leads to large improvements in throughput when the
1327          * TLB is under pressure.
1328          *
1329          * The L2 TLB distributes small and large fragments into two
1330          * asymmetric partitions. The large fragment cache is significantly
1331          * larger. Thus, we try to use large fragments wherever possible.
1332          * Userspace can support this by aligning virtual base address and
1333          * allocation size to the fragment size.
1334          *
1335          * Starting with Vega10 the fragment size only controls the L1. The L2
1336          * is now directly feed with small/huge/giant pages from the walker.
1337          */
1338         unsigned max_frag;
1339
1340         if (params->adev->asic_type < CHIP_VEGA10)
1341                 max_frag = params->adev->vm_manager.fragment_size;
1342         else
1343                 max_frag = 31;
1344
1345         /* system pages are non continuously */
1346         if (params->pages_addr) {
1347                 *frag = 0;
1348                 *frag_end = end;
1349                 return;
1350         }
1351
1352         /* This intentionally wraps around if no bit is set */
1353         *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1);
1354         if (*frag >= max_frag) {
1355                 *frag = max_frag;
1356                 *frag_end = end & ~((1ULL << max_frag) - 1);
1357         } else {
1358                 *frag_end = start + (1 << *frag);
1359         }
1360 }
1361
1362 /**
1363  * amdgpu_vm_update_ptes - make sure that page tables are valid
1364  *
1365  * @params: see amdgpu_vm_update_params definition
1366  * @start: start of GPU address range
1367  * @end: end of GPU address range
1368  * @dst: destination address to map to, the next dst inside the function
1369  * @flags: mapping flags
1370  *
1371  * Update the page tables in the range @start - @end.
1372  *
1373  * Returns:
1374  * 0 for success, -EINVAL for failure.
1375  */
1376 static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
1377                                  uint64_t start, uint64_t end,
1378                                  uint64_t dst, uint64_t flags)
1379 {
1380         struct amdgpu_device *adev = params->adev;
1381         struct amdgpu_vm_pt_cursor cursor;
1382         uint64_t frag_start = start, frag_end;
1383         unsigned int frag;
1384         int r;
1385
1386         /* figure out the initial fragment */
1387         amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end);
1388
1389         /* walk over the address space and update the PTs */
1390         amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
1391         while (cursor.pfn < end) {
1392                 unsigned shift, parent_shift, mask;
1393                 uint64_t incr, entry_end, pe_start;
1394                 struct amdgpu_bo *pt;
1395
1396                 r = amdgpu_vm_alloc_pts(params->adev, params->vm, &cursor);
1397                 if (r)
1398                         return r;
1399
1400                 pt = cursor.entry->base.bo;
1401
1402                 /* The root level can't be a huge page */
1403                 if (cursor.level == adev->vm_manager.root_level) {
1404                         if (!amdgpu_vm_pt_descendant(adev, &cursor))
1405                                 return -ENOENT;
1406                         continue;
1407                 }
1408
1409                 shift = amdgpu_vm_level_shift(adev, cursor.level);
1410                 parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
1411                 if (adev->asic_type < CHIP_VEGA10 &&
1412                     (flags & AMDGPU_PTE_VALID)) {
1413                         /* No huge page support before GMC v9 */
1414                         if (cursor.level != AMDGPU_VM_PTB) {
1415                                 if (!amdgpu_vm_pt_descendant(adev, &cursor))
1416                                         return -ENOENT;
1417                                 continue;
1418                         }
1419                 } else if (frag < shift) {
1420                         /* We can't use this level when the fragment size is
1421                          * smaller than the address shift. Go to the next
1422                          * child entry and try again.
1423                          */
1424                         if (!amdgpu_vm_pt_descendant(adev, &cursor))
1425                                 return -ENOENT;
1426                         continue;
1427                 } else if (frag >= parent_shift &&
1428                            cursor.level - 1 != adev->vm_manager.root_level) {
1429                         /* If the fragment size is even larger than the parent
1430                          * shift we should go up one level and check it again
1431                          * unless one level up is the root level.
1432                          */
1433                         if (!amdgpu_vm_pt_ancestor(&cursor))
1434                                 return -ENOENT;
1435                         continue;
1436                 }
1437
1438                 /* Looks good so far, calculate parameters for the update */
1439                 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift;
1440                 mask = amdgpu_vm_entries_mask(adev, cursor.level);
1441                 pe_start = ((cursor.pfn >> shift) & mask) * 8;
1442                 entry_end = (uint64_t)(mask + 1) << shift;
1443                 entry_end += cursor.pfn & ~(entry_end - 1);
1444                 entry_end = min(entry_end, end);
1445
1446                 do {
1447                         uint64_t upd_end = min(entry_end, frag_end);
1448                         unsigned nptes = (upd_end - frag_start) >> shift;
1449
1450                         amdgpu_vm_update_flags(params, pt, cursor.level,
1451                                                pe_start, dst, nptes, incr,
1452                                                flags | AMDGPU_PTE_FRAG(frag));
1453
1454                         pe_start += nptes * 8;
1455                         dst += (uint64_t)nptes * AMDGPU_GPU_PAGE_SIZE << shift;
1456
1457                         frag_start = upd_end;
1458                         if (frag_start >= frag_end) {
1459                                 /* figure out the next fragment */
1460                                 amdgpu_vm_fragment(params, frag_start, end,
1461                                                    flags, &frag, &frag_end);
1462                                 if (frag < shift)
1463                                         break;
1464                         }
1465                 } while (frag_start < entry_end);
1466
1467                 if (amdgpu_vm_pt_descendant(adev, &cursor)) {
1468                         /* Free all child entries */
1469                         while (cursor.pfn < frag_start) {
1470                                 amdgpu_vm_free_pts(adev, params->vm, &cursor);
1471                                 amdgpu_vm_pt_next(adev, &cursor);
1472                         }
1473
1474                 } else if (frag >= shift) {
1475                         /* or just move on to the next on the same level. */
1476                         amdgpu_vm_pt_next(adev, &cursor);
1477                 }
1478         }
1479
1480         return 0;
1481 }
1482
1483 /**
1484  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1485  *
1486  * @adev: amdgpu_device pointer
1487  * @exclusive: fence we need to sync to
1488  * @pages_addr: DMA addresses to use for mapping
1489  * @vm: requested vm
1490  * @start: start of mapped range
1491  * @last: last mapped entry
1492  * @flags: flags for the entries
1493  * @addr: addr to set the area to
1494  * @fence: optional resulting fence
1495  *
1496  * Fill in the page table entries between @start and @last.
1497  *
1498  * Returns:
1499  * 0 for success, -EINVAL for failure.
1500  */
1501 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1502                                        struct dma_fence *exclusive,
1503                                        dma_addr_t *pages_addr,
1504                                        struct amdgpu_vm *vm,
1505                                        uint64_t start, uint64_t last,
1506                                        uint64_t flags, uint64_t addr,
1507                                        struct dma_fence **fence)
1508 {
1509         struct amdgpu_vm_update_params params;
1510         void *owner = AMDGPU_FENCE_OWNER_VM;
1511         int r;
1512
1513         memset(&params, 0, sizeof(params));
1514         params.adev = adev;
1515         params.vm = vm;
1516         params.pages_addr = pages_addr;
1517
1518         /* sync to everything except eviction fences on unmapping */
1519         if (!(flags & AMDGPU_PTE_VALID))
1520                 owner = AMDGPU_FENCE_OWNER_KFD;
1521
1522         r = vm->update_funcs->prepare(&params, owner, exclusive);
1523         if (r)
1524                 return r;
1525
1526         r = amdgpu_vm_update_ptes(&params, start, last + 1, addr, flags);
1527         if (r)
1528                 return r;
1529
1530         return vm->update_funcs->commit(&params, fence);
1531 }
1532
1533 /**
1534  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1535  *
1536  * @adev: amdgpu_device pointer
1537  * @exclusive: fence we need to sync to
1538  * @pages_addr: DMA addresses to use for mapping
1539  * @vm: requested vm
1540  * @mapping: mapped range and flags to use for the update
1541  * @flags: HW flags for the mapping
1542  * @bo_adev: amdgpu_device pointer that bo actually been allocated
1543  * @nodes: array of drm_mm_nodes with the MC addresses
1544  * @fence: optional resulting fence
1545  *
1546  * Split the mapping into smaller chunks so that each update fits
1547  * into a SDMA IB.
1548  *
1549  * Returns:
1550  * 0 for success, -EINVAL for failure.
1551  */
1552 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1553                                       struct dma_fence *exclusive,
1554                                       dma_addr_t *pages_addr,
1555                                       struct amdgpu_vm *vm,
1556                                       struct amdgpu_bo_va_mapping *mapping,
1557                                       uint64_t flags,
1558                                       struct amdgpu_device *bo_adev,
1559                                       struct drm_mm_node *nodes,
1560                                       struct dma_fence **fence)
1561 {
1562         unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
1563         uint64_t pfn, start = mapping->start;
1564         int r;
1565
1566         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1567          * but in case of something, we filter the flags in first place
1568          */
1569         if (!(mapping->flags & AMDGPU_PTE_READABLE))
1570                 flags &= ~AMDGPU_PTE_READABLE;
1571         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1572                 flags &= ~AMDGPU_PTE_WRITEABLE;
1573
1574         flags &= ~AMDGPU_PTE_EXECUTABLE;
1575         flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1576
1577         if (adev->asic_type == CHIP_NAVI10) {
1578                 flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK;
1579                 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK);
1580         } else {
1581                 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1582                 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK);
1583         }
1584
1585         if ((mapping->flags & AMDGPU_PTE_PRT) &&
1586             (adev->asic_type >= CHIP_VEGA10)) {
1587                 flags |= AMDGPU_PTE_PRT;
1588                 if (adev->asic_type >= CHIP_NAVI10) {
1589                         flags |= AMDGPU_PTE_SNOOPED;
1590                         flags |= AMDGPU_PTE_LOG;
1591                         flags |= AMDGPU_PTE_SYSTEM;
1592                 }
1593                 flags &= ~AMDGPU_PTE_VALID;
1594         }
1595
1596         trace_amdgpu_vm_bo_update(mapping);
1597
1598         pfn = mapping->offset >> PAGE_SHIFT;
1599         if (nodes) {
1600                 while (pfn >= nodes->size) {
1601                         pfn -= nodes->size;
1602                         ++nodes;
1603                 }
1604         }
1605
1606         do {
1607                 dma_addr_t *dma_addr = NULL;
1608                 uint64_t max_entries;
1609                 uint64_t addr, last;
1610
1611                 if (nodes) {
1612                         addr = nodes->start << PAGE_SHIFT;
1613                         max_entries = (nodes->size - pfn) *
1614                                 AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1615                 } else {
1616                         addr = 0;
1617                         max_entries = S64_MAX;
1618                 }
1619
1620                 if (pages_addr) {
1621                         uint64_t count;
1622
1623                         for (count = 1;
1624                              count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1625                              ++count) {
1626                                 uint64_t idx = pfn + count;
1627
1628                                 if (pages_addr[idx] !=
1629                                     (pages_addr[idx - 1] + PAGE_SIZE))
1630                                         break;
1631                         }
1632
1633                         if (count < min_linear_pages) {
1634                                 addr = pfn << PAGE_SHIFT;
1635                                 dma_addr = pages_addr;
1636                         } else {
1637                                 addr = pages_addr[pfn];
1638                                 max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1639                         }
1640
1641                 } else if (flags & AMDGPU_PTE_VALID) {
1642                         addr += bo_adev->vm_manager.vram_base_offset;
1643                         addr += pfn << PAGE_SHIFT;
1644                 }
1645
1646                 last = min((uint64_t)mapping->last, start + max_entries - 1);
1647                 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
1648                                                 start, last, flags, addr,
1649                                                 fence);
1650                 if (r)
1651                         return r;
1652
1653                 pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1654                 if (nodes && nodes->size == pfn) {
1655                         pfn = 0;
1656                         ++nodes;
1657                 }
1658                 start = last + 1;
1659
1660         } while (unlikely(start != mapping->last + 1));
1661
1662         return 0;
1663 }
1664
1665 /**
1666  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1667  *
1668  * @adev: amdgpu_device pointer
1669  * @bo_va: requested BO and VM object
1670  * @clear: if true clear the entries
1671  *
1672  * Fill in the page table entries for @bo_va.
1673  *
1674  * Returns:
1675  * 0 for success, -EINVAL for failure.
1676  */
1677 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1678                         struct amdgpu_bo_va *bo_va,
1679                         bool clear)
1680 {
1681         struct amdgpu_bo *bo = bo_va->base.bo;
1682         struct amdgpu_vm *vm = bo_va->base.vm;
1683         struct amdgpu_bo_va_mapping *mapping;
1684         dma_addr_t *pages_addr = NULL;
1685         struct ttm_mem_reg *mem;
1686         struct drm_mm_node *nodes;
1687         struct dma_fence *exclusive, **last_update;
1688         uint64_t flags;
1689         struct amdgpu_device *bo_adev = adev;
1690         int r;
1691
1692         if (clear || !bo) {
1693                 mem = NULL;
1694                 nodes = NULL;
1695                 exclusive = NULL;
1696         } else {
1697                 struct ttm_dma_tt *ttm;
1698
1699                 mem = &bo->tbo.mem;
1700                 nodes = mem->mm_node;
1701                 if (mem->mem_type == TTM_PL_TT) {
1702                         ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm);
1703                         pages_addr = ttm->dma_address;
1704                 }
1705                 exclusive = reservation_object_get_excl(bo->tbo.resv);
1706         }
1707
1708         if (bo) {
1709                 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1710                 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1711         } else {
1712                 flags = 0x0;
1713         }
1714
1715         if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
1716                 last_update = &vm->last_update;
1717         else
1718                 last_update = &bo_va->last_pt_update;
1719
1720         if (!clear && bo_va->base.moved) {
1721                 bo_va->base.moved = false;
1722                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1723
1724         } else if (bo_va->cleared != clear) {
1725                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1726         }
1727
1728         list_for_each_entry(mapping, &bo_va->invalids, list) {
1729                 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
1730                                                mapping, flags, bo_adev, nodes,
1731                                                last_update);
1732                 if (r)
1733                         return r;
1734         }
1735
1736         if (vm->use_cpu_for_update) {
1737                 /* Flush HDP */
1738                 mb();
1739                 amdgpu_asic_flush_hdp(adev, NULL);
1740         }
1741
1742         /* If the BO is not in its preferred location add it back to
1743          * the evicted list so that it gets validated again on the
1744          * next command submission.
1745          */
1746         if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
1747                 uint32_t mem_type = bo->tbo.mem.mem_type;
1748
1749                 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
1750                         amdgpu_vm_bo_evicted(&bo_va->base);
1751                 else
1752                         amdgpu_vm_bo_idle(&bo_va->base);
1753         } else {
1754                 amdgpu_vm_bo_done(&bo_va->base);
1755         }
1756
1757         list_splice_init(&bo_va->invalids, &bo_va->valids);
1758         bo_va->cleared = clear;
1759
1760         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1761                 list_for_each_entry(mapping, &bo_va->valids, list)
1762                         trace_amdgpu_vm_bo_mapping(mapping);
1763         }
1764
1765         return 0;
1766 }
1767
1768 /**
1769  * amdgpu_vm_update_prt_state - update the global PRT state
1770  *
1771  * @adev: amdgpu_device pointer
1772  */
1773 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1774 {
1775         unsigned long flags;
1776         bool enable;
1777
1778         spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1779         enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1780         adev->gmc.gmc_funcs->set_prt(adev, enable);
1781         spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1782 }
1783
1784 /**
1785  * amdgpu_vm_prt_get - add a PRT user
1786  *
1787  * @adev: amdgpu_device pointer
1788  */
1789 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1790 {
1791         if (!adev->gmc.gmc_funcs->set_prt)
1792                 return;
1793
1794         if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1795                 amdgpu_vm_update_prt_state(adev);
1796 }
1797
1798 /**
1799  * amdgpu_vm_prt_put - drop a PRT user
1800  *
1801  * @adev: amdgpu_device pointer
1802  */
1803 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1804 {
1805         if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1806                 amdgpu_vm_update_prt_state(adev);
1807 }
1808
1809 /**
1810  * amdgpu_vm_prt_cb - callback for updating the PRT status
1811  *
1812  * @fence: fence for the callback
1813  * @_cb: the callback function
1814  */
1815 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1816 {
1817         struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1818
1819         amdgpu_vm_prt_put(cb->adev);
1820         kfree(cb);
1821 }
1822
1823 /**
1824  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1825  *
1826  * @adev: amdgpu_device pointer
1827  * @fence: fence for the callback
1828  */
1829 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1830                                  struct dma_fence *fence)
1831 {
1832         struct amdgpu_prt_cb *cb;
1833
1834         if (!adev->gmc.gmc_funcs->set_prt)
1835                 return;
1836
1837         cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1838         if (!cb) {
1839                 /* Last resort when we are OOM */
1840                 if (fence)
1841                         dma_fence_wait(fence, false);
1842
1843                 amdgpu_vm_prt_put(adev);
1844         } else {
1845                 cb->adev = adev;
1846                 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1847                                                      amdgpu_vm_prt_cb))
1848                         amdgpu_vm_prt_cb(fence, &cb->cb);
1849         }
1850 }
1851
1852 /**
1853  * amdgpu_vm_free_mapping - free a mapping
1854  *
1855  * @adev: amdgpu_device pointer
1856  * @vm: requested vm
1857  * @mapping: mapping to be freed
1858  * @fence: fence of the unmap operation
1859  *
1860  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1861  */
1862 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1863                                    struct amdgpu_vm *vm,
1864                                    struct amdgpu_bo_va_mapping *mapping,
1865                                    struct dma_fence *fence)
1866 {
1867         if (mapping->flags & AMDGPU_PTE_PRT)
1868                 amdgpu_vm_add_prt_cb(adev, fence);
1869         kfree(mapping);
1870 }
1871
1872 /**
1873  * amdgpu_vm_prt_fini - finish all prt mappings
1874  *
1875  * @adev: amdgpu_device pointer
1876  * @vm: requested vm
1877  *
1878  * Register a cleanup callback to disable PRT support after VM dies.
1879  */
1880 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1881 {
1882         struct reservation_object *resv = vm->root.base.bo->tbo.resv;
1883         struct dma_fence *excl, **shared;
1884         unsigned i, shared_count;
1885         int r;
1886
1887         r = reservation_object_get_fences_rcu(resv, &excl,
1888                                               &shared_count, &shared);
1889         if (r) {
1890                 /* Not enough memory to grab the fence list, as last resort
1891                  * block for all the fences to complete.
1892                  */
1893                 reservation_object_wait_timeout_rcu(resv, true, false,
1894                                                     MAX_SCHEDULE_TIMEOUT);
1895                 return;
1896         }
1897
1898         /* Add a callback for each fence in the reservation object */
1899         amdgpu_vm_prt_get(adev);
1900         amdgpu_vm_add_prt_cb(adev, excl);
1901
1902         for (i = 0; i < shared_count; ++i) {
1903                 amdgpu_vm_prt_get(adev);
1904                 amdgpu_vm_add_prt_cb(adev, shared[i]);
1905         }
1906
1907         kfree(shared);
1908 }
1909
1910 /**
1911  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1912  *
1913  * @adev: amdgpu_device pointer
1914  * @vm: requested vm
1915  * @fence: optional resulting fence (unchanged if no work needed to be done
1916  * or if an error occurred)
1917  *
1918  * Make sure all freed BOs are cleared in the PT.
1919  * PTs have to be reserved and mutex must be locked!
1920  *
1921  * Returns:
1922  * 0 for success.
1923  *
1924  */
1925 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1926                           struct amdgpu_vm *vm,
1927                           struct dma_fence **fence)
1928 {
1929         struct amdgpu_bo_va_mapping *mapping;
1930         uint64_t init_pte_value = 0;
1931         struct dma_fence *f = NULL;
1932         int r;
1933
1934         while (!list_empty(&vm->freed)) {
1935                 mapping = list_first_entry(&vm->freed,
1936                         struct amdgpu_bo_va_mapping, list);
1937                 list_del(&mapping->list);
1938
1939                 if (vm->pte_support_ats &&
1940                     mapping->start < AMDGPU_GMC_HOLE_START)
1941                         init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1942
1943                 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
1944                                                 mapping->start, mapping->last,
1945                                                 init_pte_value, 0, &f);
1946                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
1947                 if (r) {
1948                         dma_fence_put(f);
1949                         return r;
1950                 }
1951         }
1952
1953         if (fence && f) {
1954                 dma_fence_put(*fence);
1955                 *fence = f;
1956         } else {
1957                 dma_fence_put(f);
1958         }
1959
1960         return 0;
1961
1962 }
1963
1964 /**
1965  * amdgpu_vm_handle_moved - handle moved BOs in the PT
1966  *
1967  * @adev: amdgpu_device pointer
1968  * @vm: requested vm
1969  *
1970  * Make sure all BOs which are moved are updated in the PTs.
1971  *
1972  * Returns:
1973  * 0 for success.
1974  *
1975  * PTs have to be reserved!
1976  */
1977 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1978                            struct amdgpu_vm *vm)
1979 {
1980         struct amdgpu_bo_va *bo_va, *tmp;
1981         struct reservation_object *resv;
1982         bool clear;
1983         int r;
1984
1985         list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
1986                 /* Per VM BOs never need to bo cleared in the page tables */
1987                 r = amdgpu_vm_bo_update(adev, bo_va, false);
1988                 if (r)
1989                         return r;
1990         }
1991
1992         spin_lock(&vm->invalidated_lock);
1993         while (!list_empty(&vm->invalidated)) {
1994                 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1995                                          base.vm_status);
1996                 resv = bo_va->base.bo->tbo.resv;
1997                 spin_unlock(&vm->invalidated_lock);
1998
1999                 /* Try to reserve the BO to avoid clearing its ptes */
2000                 if (!amdgpu_vm_debug && reservation_object_trylock(resv))
2001                         clear = false;
2002                 /* Somebody else is using the BO right now */
2003                 else
2004                         clear = true;
2005
2006                 r = amdgpu_vm_bo_update(adev, bo_va, clear);
2007                 if (r)
2008                         return r;
2009
2010                 if (!clear)
2011                         reservation_object_unlock(resv);
2012                 spin_lock(&vm->invalidated_lock);
2013         }
2014         spin_unlock(&vm->invalidated_lock);
2015
2016         return 0;
2017 }
2018
2019 /**
2020  * amdgpu_vm_bo_add - add a bo to a specific vm
2021  *
2022  * @adev: amdgpu_device pointer
2023  * @vm: requested vm
2024  * @bo: amdgpu buffer object
2025  *
2026  * Add @bo into the requested vm.
2027  * Add @bo to the list of bos associated with the vm
2028  *
2029  * Returns:
2030  * Newly added bo_va or NULL for failure
2031  *
2032  * Object has to be reserved!
2033  */
2034 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
2035                                       struct amdgpu_vm *vm,
2036                                       struct amdgpu_bo *bo)
2037 {
2038         struct amdgpu_bo_va *bo_va;
2039
2040         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
2041         if (bo_va == NULL) {
2042                 return NULL;
2043         }
2044         amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
2045
2046         bo_va->ref_count = 1;
2047         INIT_LIST_HEAD(&bo_va->valids);
2048         INIT_LIST_HEAD(&bo_va->invalids);
2049
2050         if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) &&
2051             (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
2052                 bo_va->is_xgmi = true;
2053                 mutex_lock(&adev->vm_manager.lock_pstate);
2054                 /* Power up XGMI if it can be potentially used */
2055                 if (++adev->vm_manager.xgmi_map_counter == 1)
2056                         amdgpu_xgmi_set_pstate(adev, 1);
2057                 mutex_unlock(&adev->vm_manager.lock_pstate);
2058         }
2059
2060         return bo_va;
2061 }
2062
2063
2064 /**
2065  * amdgpu_vm_bo_insert_mapping - insert a new mapping
2066  *
2067  * @adev: amdgpu_device pointer
2068  * @bo_va: bo_va to store the address
2069  * @mapping: the mapping to insert
2070  *
2071  * Insert a new mapping into all structures.
2072  */
2073 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
2074                                     struct amdgpu_bo_va *bo_va,
2075                                     struct amdgpu_bo_va_mapping *mapping)
2076 {
2077         struct amdgpu_vm *vm = bo_va->base.vm;
2078         struct amdgpu_bo *bo = bo_va->base.bo;
2079
2080         mapping->bo_va = bo_va;
2081         list_add(&mapping->list, &bo_va->invalids);
2082         amdgpu_vm_it_insert(mapping, &vm->va);
2083
2084         if (mapping->flags & AMDGPU_PTE_PRT)
2085                 amdgpu_vm_prt_get(adev);
2086
2087         if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
2088             !bo_va->base.moved) {
2089                 list_move(&bo_va->base.vm_status, &vm->moved);
2090         }
2091         trace_amdgpu_vm_bo_map(bo_va, mapping);
2092 }
2093
2094 /**
2095  * amdgpu_vm_bo_map - map bo inside a vm
2096  *
2097  * @adev: amdgpu_device pointer
2098  * @bo_va: bo_va to store the address
2099  * @saddr: where to map the BO
2100  * @offset: requested offset in the BO
2101  * @size: BO size in bytes
2102  * @flags: attributes of pages (read/write/valid/etc.)
2103  *
2104  * Add a mapping of the BO at the specefied addr into the VM.
2105  *
2106  * Returns:
2107  * 0 for success, error for failure.
2108  *
2109  * Object has to be reserved and unreserved outside!
2110  */
2111 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
2112                      struct amdgpu_bo_va *bo_va,
2113                      uint64_t saddr, uint64_t offset,
2114                      uint64_t size, uint64_t flags)
2115 {
2116         struct amdgpu_bo_va_mapping *mapping, *tmp;
2117         struct amdgpu_bo *bo = bo_va->base.bo;
2118         struct amdgpu_vm *vm = bo_va->base.vm;
2119         uint64_t eaddr;
2120
2121         /* validate the parameters */
2122         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2123             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2124                 return -EINVAL;
2125
2126         /* make sure object fit at this offset */
2127         eaddr = saddr + size - 1;
2128         if (saddr >= eaddr ||
2129             (bo && offset + size > amdgpu_bo_size(bo)))
2130                 return -EINVAL;
2131
2132         saddr /= AMDGPU_GPU_PAGE_SIZE;
2133         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2134
2135         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2136         if (tmp) {
2137                 /* bo and tmp overlap, invalid addr */
2138                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
2139                         "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
2140                         tmp->start, tmp->last + 1);
2141                 return -EINVAL;
2142         }
2143
2144         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2145         if (!mapping)
2146                 return -ENOMEM;
2147
2148         mapping->start = saddr;
2149         mapping->last = eaddr;
2150         mapping->offset = offset;
2151         mapping->flags = flags;
2152
2153         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2154
2155         return 0;
2156 }
2157
2158 /**
2159  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2160  *
2161  * @adev: amdgpu_device pointer
2162  * @bo_va: bo_va to store the address
2163  * @saddr: where to map the BO
2164  * @offset: requested offset in the BO
2165  * @size: BO size in bytes
2166  * @flags: attributes of pages (read/write/valid/etc.)
2167  *
2168  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2169  * mappings as we do so.
2170  *
2171  * Returns:
2172  * 0 for success, error for failure.
2173  *
2174  * Object has to be reserved and unreserved outside!
2175  */
2176 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
2177                              struct amdgpu_bo_va *bo_va,
2178                              uint64_t saddr, uint64_t offset,
2179                              uint64_t size, uint64_t flags)
2180 {
2181         struct amdgpu_bo_va_mapping *mapping;
2182         struct amdgpu_bo *bo = bo_va->base.bo;
2183         uint64_t eaddr;
2184         int r;
2185
2186         /* validate the parameters */
2187         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2188             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2189                 return -EINVAL;
2190
2191         /* make sure object fit at this offset */
2192         eaddr = saddr + size - 1;
2193         if (saddr >= eaddr ||
2194             (bo && offset + size > amdgpu_bo_size(bo)))
2195                 return -EINVAL;
2196
2197         /* Allocate all the needed memory */
2198         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2199         if (!mapping)
2200                 return -ENOMEM;
2201
2202         r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
2203         if (r) {
2204                 kfree(mapping);
2205                 return r;
2206         }
2207
2208         saddr /= AMDGPU_GPU_PAGE_SIZE;
2209         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2210
2211         mapping->start = saddr;
2212         mapping->last = eaddr;
2213         mapping->offset = offset;
2214         mapping->flags = flags;
2215
2216         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2217
2218         return 0;
2219 }
2220
2221 /**
2222  * amdgpu_vm_bo_unmap - remove bo mapping from vm
2223  *
2224  * @adev: amdgpu_device pointer
2225  * @bo_va: bo_va to remove the address from
2226  * @saddr: where to the BO is mapped
2227  *
2228  * Remove a mapping of the BO at the specefied addr from the VM.
2229  *
2230  * Returns:
2231  * 0 for success, error for failure.
2232  *
2233  * Object has to be reserved and unreserved outside!
2234  */
2235 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2236                        struct amdgpu_bo_va *bo_va,
2237                        uint64_t saddr)
2238 {
2239         struct amdgpu_bo_va_mapping *mapping;
2240         struct amdgpu_vm *vm = bo_va->base.vm;
2241         bool valid = true;
2242
2243         saddr /= AMDGPU_GPU_PAGE_SIZE;
2244
2245         list_for_each_entry(mapping, &bo_va->valids, list) {
2246                 if (mapping->start == saddr)
2247                         break;
2248         }
2249
2250         if (&mapping->list == &bo_va->valids) {
2251                 valid = false;
2252
2253                 list_for_each_entry(mapping, &bo_va->invalids, list) {
2254                         if (mapping->start == saddr)
2255                                 break;
2256                 }
2257
2258                 if (&mapping->list == &bo_va->invalids)
2259                         return -ENOENT;
2260         }
2261
2262         list_del(&mapping->list);
2263         amdgpu_vm_it_remove(mapping, &vm->va);
2264         mapping->bo_va = NULL;
2265         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2266
2267         if (valid)
2268                 list_add(&mapping->list, &vm->freed);
2269         else
2270                 amdgpu_vm_free_mapping(adev, vm, mapping,
2271                                        bo_va->last_pt_update);
2272
2273         return 0;
2274 }
2275
2276 /**
2277  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2278  *
2279  * @adev: amdgpu_device pointer
2280  * @vm: VM structure to use
2281  * @saddr: start of the range
2282  * @size: size of the range
2283  *
2284  * Remove all mappings in a range, split them as appropriate.
2285  *
2286  * Returns:
2287  * 0 for success, error for failure.
2288  */
2289 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2290                                 struct amdgpu_vm *vm,
2291                                 uint64_t saddr, uint64_t size)
2292 {
2293         struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2294         LIST_HEAD(removed);
2295         uint64_t eaddr;
2296
2297         eaddr = saddr + size - 1;
2298         saddr /= AMDGPU_GPU_PAGE_SIZE;
2299         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2300
2301         /* Allocate all the needed memory */
2302         before = kzalloc(sizeof(*before), GFP_KERNEL);
2303         if (!before)
2304                 return -ENOMEM;
2305         INIT_LIST_HEAD(&before->list);
2306
2307         after = kzalloc(sizeof(*after), GFP_KERNEL);
2308         if (!after) {
2309                 kfree(before);
2310                 return -ENOMEM;
2311         }
2312         INIT_LIST_HEAD(&after->list);
2313
2314         /* Now gather all removed mappings */
2315         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2316         while (tmp) {
2317                 /* Remember mapping split at the start */
2318                 if (tmp->start < saddr) {
2319                         before->start = tmp->start;
2320                         before->last = saddr - 1;
2321                         before->offset = tmp->offset;
2322                         before->flags = tmp->flags;
2323                         before->bo_va = tmp->bo_va;
2324                         list_add(&before->list, &tmp->bo_va->invalids);
2325                 }
2326
2327                 /* Remember mapping split at the end */
2328                 if (tmp->last > eaddr) {
2329                         after->start = eaddr + 1;
2330                         after->last = tmp->last;
2331                         after->offset = tmp->offset;
2332                         after->offset += after->start - tmp->start;
2333                         after->flags = tmp->flags;
2334                         after->bo_va = tmp->bo_va;
2335                         list_add(&after->list, &tmp->bo_va->invalids);
2336                 }
2337
2338                 list_del(&tmp->list);
2339                 list_add(&tmp->list, &removed);
2340
2341                 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2342         }
2343
2344         /* And free them up */
2345         list_for_each_entry_safe(tmp, next, &removed, list) {
2346                 amdgpu_vm_it_remove(tmp, &vm->va);
2347                 list_del(&tmp->list);
2348
2349                 if (tmp->start < saddr)
2350                     tmp->start = saddr;
2351                 if (tmp->last > eaddr)
2352                     tmp->last = eaddr;
2353
2354                 tmp->bo_va = NULL;
2355                 list_add(&tmp->list, &vm->freed);
2356                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2357         }
2358
2359         /* Insert partial mapping before the range */
2360         if (!list_empty(&before->list)) {
2361                 amdgpu_vm_it_insert(before, &vm->va);
2362                 if (before->flags & AMDGPU_PTE_PRT)
2363                         amdgpu_vm_prt_get(adev);
2364         } else {
2365                 kfree(before);
2366         }
2367
2368         /* Insert partial mapping after the range */
2369         if (!list_empty(&after->list)) {
2370                 amdgpu_vm_it_insert(after, &vm->va);
2371                 if (after->flags & AMDGPU_PTE_PRT)
2372                         amdgpu_vm_prt_get(adev);
2373         } else {
2374                 kfree(after);
2375         }
2376
2377         return 0;
2378 }
2379
2380 /**
2381  * amdgpu_vm_bo_lookup_mapping - find mapping by address
2382  *
2383  * @vm: the requested VM
2384  * @addr: the address
2385  *
2386  * Find a mapping by it's address.
2387  *
2388  * Returns:
2389  * The amdgpu_bo_va_mapping matching for addr or NULL
2390  *
2391  */
2392 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2393                                                          uint64_t addr)
2394 {
2395         return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2396 }
2397
2398 /**
2399  * amdgpu_vm_bo_trace_cs - trace all reserved mappings
2400  *
2401  * @vm: the requested vm
2402  * @ticket: CS ticket
2403  *
2404  * Trace all mappings of BOs reserved during a command submission.
2405  */
2406 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2407 {
2408         struct amdgpu_bo_va_mapping *mapping;
2409
2410         if (!trace_amdgpu_vm_bo_cs_enabled())
2411                 return;
2412
2413         for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
2414              mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
2415                 if (mapping->bo_va && mapping->bo_va->base.bo) {
2416                         struct amdgpu_bo *bo;
2417
2418                         bo = mapping->bo_va->base.bo;
2419                         if (READ_ONCE(bo->tbo.resv->lock.ctx) != ticket)
2420                                 continue;
2421                 }
2422
2423                 trace_amdgpu_vm_bo_cs(mapping);
2424         }
2425 }
2426
2427 /**
2428  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2429  *
2430  * @adev: amdgpu_device pointer
2431  * @bo_va: requested bo_va
2432  *
2433  * Remove @bo_va->bo from the requested vm.
2434  *
2435  * Object have to be reserved!
2436  */
2437 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2438                       struct amdgpu_bo_va *bo_va)
2439 {
2440         struct amdgpu_bo_va_mapping *mapping, *next;
2441         struct amdgpu_bo *bo = bo_va->base.bo;
2442         struct amdgpu_vm *vm = bo_va->base.vm;
2443         struct amdgpu_vm_bo_base **base;
2444
2445         if (bo) {
2446                 if (bo->tbo.resv == vm->root.base.bo->tbo.resv)
2447                         vm->bulk_moveable = false;
2448
2449                 for (base = &bo_va->base.bo->vm_bo; *base;
2450                      base = &(*base)->next) {
2451                         if (*base != &bo_va->base)
2452                                 continue;
2453
2454                         *base = bo_va->base.next;
2455                         break;
2456                 }
2457         }
2458
2459         spin_lock(&vm->invalidated_lock);
2460         list_del(&bo_va->base.vm_status);
2461         spin_unlock(&vm->invalidated_lock);
2462
2463         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2464                 list_del(&mapping->list);
2465                 amdgpu_vm_it_remove(mapping, &vm->va);
2466                 mapping->bo_va = NULL;
2467                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2468                 list_add(&mapping->list, &vm->freed);
2469         }
2470         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2471                 list_del(&mapping->list);
2472                 amdgpu_vm_it_remove(mapping, &vm->va);
2473                 amdgpu_vm_free_mapping(adev, vm, mapping,
2474                                        bo_va->last_pt_update);
2475         }
2476
2477         dma_fence_put(bo_va->last_pt_update);
2478
2479         if (bo && bo_va->is_xgmi) {
2480                 mutex_lock(&adev->vm_manager.lock_pstate);
2481                 if (--adev->vm_manager.xgmi_map_counter == 0)
2482                         amdgpu_xgmi_set_pstate(adev, 0);
2483                 mutex_unlock(&adev->vm_manager.lock_pstate);
2484         }
2485
2486         kfree(bo_va);
2487 }
2488
2489 /**
2490  * amdgpu_vm_bo_invalidate - mark the bo as invalid
2491  *
2492  * @adev: amdgpu_device pointer
2493  * @bo: amdgpu buffer object
2494  * @evicted: is the BO evicted
2495  *
2496  * Mark @bo as invalid.
2497  */
2498 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2499                              struct amdgpu_bo *bo, bool evicted)
2500 {
2501         struct amdgpu_vm_bo_base *bo_base;
2502
2503         /* shadow bo doesn't have bo base, its validation needs its parent */
2504         if (bo->parent && bo->parent->shadow == bo)
2505                 bo = bo->parent;
2506
2507         for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2508                 struct amdgpu_vm *vm = bo_base->vm;
2509
2510                 if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
2511                         amdgpu_vm_bo_evicted(bo_base);
2512                         continue;
2513                 }
2514
2515                 if (bo_base->moved)
2516                         continue;
2517                 bo_base->moved = true;
2518
2519                 if (bo->tbo.type == ttm_bo_type_kernel)
2520                         amdgpu_vm_bo_relocated(bo_base);
2521                 else if (bo->tbo.resv == vm->root.base.bo->tbo.resv)
2522                         amdgpu_vm_bo_moved(bo_base);
2523                 else
2524                         amdgpu_vm_bo_invalidated(bo_base);
2525         }
2526 }
2527
2528 /**
2529  * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2530  *
2531  * @vm_size: VM size
2532  *
2533  * Returns:
2534  * VM page table as power of two
2535  */
2536 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2537 {
2538         /* Total bits covered by PD + PTs */
2539         unsigned bits = ilog2(vm_size) + 18;
2540
2541         /* Make sure the PD is 4K in size up to 8GB address space.
2542            Above that split equal between PD and PTs */
2543         if (vm_size <= 8)
2544                 return (bits - 9);
2545         else
2546                 return ((bits + 3) / 2);
2547 }
2548
2549 /**
2550  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2551  *
2552  * @adev: amdgpu_device pointer
2553  * @min_vm_size: the minimum vm size in GB if it's set auto
2554  * @fragment_size_default: Default PTE fragment size
2555  * @max_level: max VMPT level
2556  * @max_bits: max address space size in bits
2557  *
2558  */
2559 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2560                            uint32_t fragment_size_default, unsigned max_level,
2561                            unsigned max_bits)
2562 {
2563         unsigned int max_size = 1 << (max_bits - 30);
2564         unsigned int vm_size;
2565         uint64_t tmp;
2566
2567         /* adjust vm size first */
2568         if (amdgpu_vm_size != -1) {
2569                 vm_size = amdgpu_vm_size;
2570                 if (vm_size > max_size) {
2571                         dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2572                                  amdgpu_vm_size, max_size);
2573                         vm_size = max_size;
2574                 }
2575         } else {
2576                 struct sysinfo si;
2577                 unsigned int phys_ram_gb;
2578
2579                 /* Optimal VM size depends on the amount of physical
2580                  * RAM available. Underlying requirements and
2581                  * assumptions:
2582                  *
2583                  *  - Need to map system memory and VRAM from all GPUs
2584                  *     - VRAM from other GPUs not known here
2585                  *     - Assume VRAM <= system memory
2586                  *  - On GFX8 and older, VM space can be segmented for
2587                  *    different MTYPEs
2588                  *  - Need to allow room for fragmentation, guard pages etc.
2589                  *
2590                  * This adds up to a rough guess of system memory x3.
2591                  * Round up to power of two to maximize the available
2592                  * VM size with the given page table size.
2593                  */
2594                 si_meminfo(&si);
2595                 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2596                                (1 << 30) - 1) >> 30;
2597                 vm_size = roundup_pow_of_two(
2598                         min(max(phys_ram_gb * 3, min_vm_size), max_size));
2599         }
2600
2601         adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2602
2603         tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2604         if (amdgpu_vm_block_size != -1)
2605                 tmp >>= amdgpu_vm_block_size - 9;
2606         tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2607         adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2608         switch (adev->vm_manager.num_level) {
2609         case 3:
2610                 adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2611                 break;
2612         case 2:
2613                 adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2614                 break;
2615         case 1:
2616                 adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2617                 break;
2618         default:
2619                 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2620         }
2621         /* block size depends on vm size and hw setup*/
2622         if (amdgpu_vm_block_size != -1)
2623                 adev->vm_manager.block_size =
2624                         min((unsigned)amdgpu_vm_block_size, max_bits
2625                             - AMDGPU_GPU_PAGE_SHIFT
2626                             - 9 * adev->vm_manager.num_level);
2627         else if (adev->vm_manager.num_level > 1)
2628                 adev->vm_manager.block_size = 9;
2629         else
2630                 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2631
2632         if (amdgpu_vm_fragment_size == -1)
2633                 adev->vm_manager.fragment_size = fragment_size_default;
2634         else
2635                 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2636
2637         DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2638                  vm_size, adev->vm_manager.num_level + 1,
2639                  adev->vm_manager.block_size,
2640                  adev->vm_manager.fragment_size);
2641 }
2642
2643 /**
2644  * amdgpu_vm_wait_idle - wait for the VM to become idle
2645  *
2646  * @vm: VM object to wait for
2647  * @timeout: timeout to wait for VM to become idle
2648  */
2649 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2650 {
2651         return reservation_object_wait_timeout_rcu(vm->root.base.bo->tbo.resv,
2652                                                    true, true, timeout);
2653 }
2654
2655 /**
2656  * amdgpu_vm_init - initialize a vm instance
2657  *
2658  * @adev: amdgpu_device pointer
2659  * @vm: requested vm
2660  * @vm_context: Indicates if it GFX or Compute context
2661  * @pasid: Process address space identifier
2662  *
2663  * Init @vm fields.
2664  *
2665  * Returns:
2666  * 0 for success, error for failure.
2667  */
2668 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2669                    int vm_context, unsigned int pasid)
2670 {
2671         struct amdgpu_bo_param bp;
2672         struct amdgpu_bo *root;
2673         int r, i;
2674
2675         vm->va = RB_ROOT_CACHED;
2676         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2677                 vm->reserved_vmid[i] = NULL;
2678         INIT_LIST_HEAD(&vm->evicted);
2679         INIT_LIST_HEAD(&vm->relocated);
2680         INIT_LIST_HEAD(&vm->moved);
2681         INIT_LIST_HEAD(&vm->idle);
2682         INIT_LIST_HEAD(&vm->invalidated);
2683         spin_lock_init(&vm->invalidated_lock);
2684         INIT_LIST_HEAD(&vm->freed);
2685
2686         /* create scheduler entity for page table updates */
2687         r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs,
2688                                   adev->vm_manager.vm_pte_num_rqs, NULL);
2689         if (r)
2690                 return r;
2691
2692         vm->pte_support_ats = false;
2693
2694         if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2695                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2696                                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2697
2698                 if (adev->asic_type == CHIP_RAVEN)
2699                         vm->pte_support_ats = true;
2700         } else {
2701                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2702                                                 AMDGPU_VM_USE_CPU_FOR_GFX);
2703         }
2704         DRM_DEBUG_DRIVER("VM update mode is %s\n",
2705                          vm->use_cpu_for_update ? "CPU" : "SDMA");
2706         WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2707                   "CPU update of VM recommended only for large BAR system\n");
2708
2709         if (vm->use_cpu_for_update)
2710                 vm->update_funcs = &amdgpu_vm_cpu_funcs;
2711         else
2712                 vm->update_funcs = &amdgpu_vm_sdma_funcs;
2713         vm->last_update = NULL;
2714
2715         amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp);
2716         if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
2717                 bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW;
2718         r = amdgpu_bo_create(adev, &bp, &root);
2719         if (r)
2720                 goto error_free_sched_entity;
2721
2722         r = amdgpu_bo_reserve(root, true);
2723         if (r)
2724                 goto error_free_root;
2725
2726         r = reservation_object_reserve_shared(root->tbo.resv, 1);
2727         if (r)
2728                 goto error_unreserve;
2729
2730         amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
2731
2732         r = amdgpu_vm_clear_bo(adev, vm, root);
2733         if (r)
2734                 goto error_unreserve;
2735
2736         amdgpu_bo_unreserve(vm->root.base.bo);
2737
2738         if (pasid) {
2739                 unsigned long flags;
2740
2741                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2742                 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2743                               GFP_ATOMIC);
2744                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2745                 if (r < 0)
2746                         goto error_free_root;
2747
2748                 vm->pasid = pasid;
2749         }
2750
2751         INIT_KFIFO(vm->faults);
2752
2753         return 0;
2754
2755 error_unreserve:
2756         amdgpu_bo_unreserve(vm->root.base.bo);
2757
2758 error_free_root:
2759         amdgpu_bo_unref(&vm->root.base.bo->shadow);
2760         amdgpu_bo_unref(&vm->root.base.bo);
2761         vm->root.base.bo = NULL;
2762
2763 error_free_sched_entity:
2764         drm_sched_entity_destroy(&vm->entity);
2765
2766         return r;
2767 }
2768
2769 /**
2770  * amdgpu_vm_check_clean_reserved - check if a VM is clean
2771  *
2772  * @adev: amdgpu_device pointer
2773  * @vm: the VM to check
2774  *
2775  * check all entries of the root PD, if any subsequent PDs are allocated,
2776  * it means there are page table creating and filling, and is no a clean
2777  * VM
2778  *
2779  * Returns:
2780  *      0 if this VM is clean
2781  */
2782 static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev,
2783         struct amdgpu_vm *vm)
2784 {
2785         enum amdgpu_vm_level root = adev->vm_manager.root_level;
2786         unsigned int entries = amdgpu_vm_num_entries(adev, root);
2787         unsigned int i = 0;
2788
2789         if (!(vm->root.entries))
2790                 return 0;
2791
2792         for (i = 0; i < entries; i++) {
2793                 if (vm->root.entries[i].base.bo)
2794                         return -EINVAL;
2795         }
2796
2797         return 0;
2798 }
2799
2800 /**
2801  * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2802  *
2803  * @adev: amdgpu_device pointer
2804  * @vm: requested vm
2805  *
2806  * This only works on GFX VMs that don't have any BOs added and no
2807  * page tables allocated yet.
2808  *
2809  * Changes the following VM parameters:
2810  * - use_cpu_for_update
2811  * - pte_supports_ats
2812  * - pasid (old PASID is released, because compute manages its own PASIDs)
2813  *
2814  * Reinitializes the page directory to reflect the changed ATS
2815  * setting.
2816  *
2817  * Returns:
2818  * 0 for success, -errno for errors.
2819  */
2820 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid)
2821 {
2822         bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2823         int r;
2824
2825         r = amdgpu_bo_reserve(vm->root.base.bo, true);
2826         if (r)
2827                 return r;
2828
2829         /* Sanity checks */
2830         r = amdgpu_vm_check_clean_reserved(adev, vm);
2831         if (r)
2832                 goto unreserve_bo;
2833
2834         if (pasid) {
2835                 unsigned long flags;
2836
2837                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2838                 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2839                               GFP_ATOMIC);
2840                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2841
2842                 if (r == -ENOSPC)
2843                         goto unreserve_bo;
2844                 r = 0;
2845         }
2846
2847         /* Check if PD needs to be reinitialized and do it before
2848          * changing any other state, in case it fails.
2849          */
2850         if (pte_support_ats != vm->pte_support_ats) {
2851                 vm->pte_support_ats = pte_support_ats;
2852                 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo);
2853                 if (r)
2854                         goto free_idr;
2855         }
2856
2857         /* Update VM state */
2858         vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2859                                     AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2860         DRM_DEBUG_DRIVER("VM update mode is %s\n",
2861                          vm->use_cpu_for_update ? "CPU" : "SDMA");
2862         WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2863                   "CPU update of VM recommended only for large BAR system\n");
2864
2865         if (vm->pasid) {
2866                 unsigned long flags;
2867
2868                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2869                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2870                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2871
2872                 /* Free the original amdgpu allocated pasid
2873                  * Will be replaced with kfd allocated pasid
2874                  */
2875                 amdgpu_pasid_free(vm->pasid);
2876                 vm->pasid = 0;
2877         }
2878
2879         /* Free the shadow bo for compute VM */
2880         amdgpu_bo_unref(&vm->root.base.bo->shadow);
2881
2882         if (pasid)
2883                 vm->pasid = pasid;
2884
2885         goto unreserve_bo;
2886
2887 free_idr:
2888         if (pasid) {
2889                 unsigned long flags;
2890
2891                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2892                 idr_remove(&adev->vm_manager.pasid_idr, pasid);
2893                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2894         }
2895 unreserve_bo:
2896         amdgpu_bo_unreserve(vm->root.base.bo);
2897         return r;
2898 }
2899
2900 /**
2901  * amdgpu_vm_release_compute - release a compute vm
2902  * @adev: amdgpu_device pointer
2903  * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
2904  *
2905  * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
2906  * pasid from vm. Compute should stop use of vm after this call.
2907  */
2908 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2909 {
2910         if (vm->pasid) {
2911                 unsigned long flags;
2912
2913                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2914                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2915                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2916         }
2917         vm->pasid = 0;
2918 }
2919
2920 /**
2921  * amdgpu_vm_fini - tear down a vm instance
2922  *
2923  * @adev: amdgpu_device pointer
2924  * @vm: requested vm
2925  *
2926  * Tear down @vm.
2927  * Unbind the VM and remove all bos from the vm bo list
2928  */
2929 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2930 {
2931         struct amdgpu_bo_va_mapping *mapping, *tmp;
2932         bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2933         struct amdgpu_bo *root;
2934         int i, r;
2935
2936         amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2937
2938         if (vm->pasid) {
2939                 unsigned long flags;
2940
2941                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2942                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2943                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2944         }
2945
2946         drm_sched_entity_destroy(&vm->entity);
2947
2948         if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2949                 dev_err(adev->dev, "still active bo inside vm\n");
2950         }
2951         rbtree_postorder_for_each_entry_safe(mapping, tmp,
2952                                              &vm->va.rb_root, rb) {
2953                 /* Don't remove the mapping here, we don't want to trigger a
2954                  * rebalance and the tree is about to be destroyed anyway.
2955                  */
2956                 list_del(&mapping->list);
2957                 kfree(mapping);
2958         }
2959         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2960                 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2961                         amdgpu_vm_prt_fini(adev, vm);
2962                         prt_fini_needed = false;
2963                 }
2964
2965                 list_del(&mapping->list);
2966                 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2967         }
2968
2969         root = amdgpu_bo_ref(vm->root.base.bo);
2970         r = amdgpu_bo_reserve(root, true);
2971         if (r) {
2972                 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
2973         } else {
2974                 amdgpu_vm_free_pts(adev, vm, NULL);
2975                 amdgpu_bo_unreserve(root);
2976         }
2977         amdgpu_bo_unref(&root);
2978         WARN_ON(vm->root.base.bo);
2979         dma_fence_put(vm->last_update);
2980         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2981                 amdgpu_vmid_free_reserved(adev, vm, i);
2982 }
2983
2984 /**
2985  * amdgpu_vm_manager_init - init the VM manager
2986  *
2987  * @adev: amdgpu_device pointer
2988  *
2989  * Initialize the VM manager structures
2990  */
2991 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2992 {
2993         unsigned i;
2994
2995         amdgpu_vmid_mgr_init(adev);
2996
2997         adev->vm_manager.fence_context =
2998                 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2999         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
3000                 adev->vm_manager.seqno[i] = 0;
3001
3002         spin_lock_init(&adev->vm_manager.prt_lock);
3003         atomic_set(&adev->vm_manager.num_prt_users, 0);
3004
3005         /* If not overridden by the user, by default, only in large BAR systems
3006          * Compute VM tables will be updated by CPU
3007          */
3008 #ifdef CONFIG_X86_64
3009         if (amdgpu_vm_update_mode == -1) {
3010                 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
3011                         adev->vm_manager.vm_update_mode =
3012                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE;
3013                 else
3014                         adev->vm_manager.vm_update_mode = 0;
3015         } else
3016                 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
3017 #else
3018         adev->vm_manager.vm_update_mode = 0;
3019 #endif
3020
3021         idr_init(&adev->vm_manager.pasid_idr);
3022         spin_lock_init(&adev->vm_manager.pasid_lock);
3023
3024         adev->vm_manager.xgmi_map_counter = 0;
3025         mutex_init(&adev->vm_manager.lock_pstate);
3026 }
3027
3028 /**
3029  * amdgpu_vm_manager_fini - cleanup VM manager
3030  *
3031  * @adev: amdgpu_device pointer
3032  *
3033  * Cleanup the VM manager and free resources.
3034  */
3035 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
3036 {
3037         WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
3038         idr_destroy(&adev->vm_manager.pasid_idr);
3039
3040         amdgpu_vmid_mgr_fini(adev);
3041 }
3042
3043 /**
3044  * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
3045  *
3046  * @dev: drm device pointer
3047  * @data: drm_amdgpu_vm
3048  * @filp: drm file pointer
3049  *
3050  * Returns:
3051  * 0 for success, -errno for errors.
3052  */
3053 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
3054 {
3055         union drm_amdgpu_vm *args = data;
3056         struct amdgpu_device *adev = dev->dev_private;
3057         struct amdgpu_fpriv *fpriv = filp->driver_priv;
3058         int r;
3059
3060         switch (args->in.op) {
3061         case AMDGPU_VM_OP_RESERVE_VMID:
3062                 /* current, we only have requirement to reserve vmid from gfxhub */
3063                 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
3064                 if (r)
3065                         return r;
3066                 break;
3067         case AMDGPU_VM_OP_UNRESERVE_VMID:
3068                 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
3069                 break;
3070         default:
3071                 return -EINVAL;
3072         }
3073
3074         return 0;
3075 }
3076
3077 /**
3078  * amdgpu_vm_get_task_info - Extracts task info for a PASID.
3079  *
3080  * @adev: drm device pointer
3081  * @pasid: PASID identifier for VM
3082  * @task_info: task_info to fill.
3083  */
3084 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
3085                          struct amdgpu_task_info *task_info)
3086 {
3087         struct amdgpu_vm *vm;
3088         unsigned long flags;
3089
3090         spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
3091
3092         vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
3093         if (vm)
3094                 *task_info = vm->task_info;
3095
3096         spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
3097 }
3098
3099 /**
3100  * amdgpu_vm_set_task_info - Sets VMs task info.
3101  *
3102  * @vm: vm for which to set the info
3103  */
3104 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
3105 {
3106         if (!vm->task_info.pid) {
3107                 vm->task_info.pid = current->pid;
3108                 get_task_comm(vm->task_info.task_name, current);
3109
3110                 if (current->group_leader->mm == current->mm) {
3111                         vm->task_info.tgid = current->group_leader->pid;
3112                         get_task_comm(vm->task_info.process_name, current->group_leader);
3113                 }
3114         }
3115 }