2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
32 #include <linux/list.h>
33 #include <linux/slab.h>
35 #include <drm/amdgpu_drm.h>
36 #include <drm/drm_cache.h>
38 #include "amdgpu_trace.h"
40 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
42 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
45 bo = container_of(tbo, struct amdgpu_bo, tbo);
49 if (bo->gem_base.import_attach)
50 drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
51 drm_gem_object_release(&bo->gem_base);
52 amdgpu_bo_unref(&bo->parent);
53 if (!list_empty(&bo->shadow_list)) {
54 mutex_lock(&adev->shadow_list_lock);
55 list_del_init(&bo->shadow_list);
56 mutex_unlock(&adev->shadow_list_lock);
62 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
64 if (bo->destroy == &amdgpu_ttm_bo_destroy)
69 static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
70 struct ttm_placement *placement,
71 struct ttm_place *places,
72 u32 domain, u64 flags)
76 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
77 unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
81 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
84 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
85 places[c].lpfn = visible_pfn;
87 places[c].flags |= TTM_PL_FLAG_TOPDOWN;
89 if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
90 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
94 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
96 if (flags & AMDGPU_GEM_CREATE_SHADOW)
97 places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT;
100 places[c].flags = TTM_PL_FLAG_TT;
101 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
102 places[c].flags |= TTM_PL_FLAG_WC |
103 TTM_PL_FLAG_UNCACHED;
105 places[c].flags |= TTM_PL_FLAG_CACHED;
109 if (domain & AMDGPU_GEM_DOMAIN_CPU) {
112 places[c].flags = TTM_PL_FLAG_SYSTEM;
113 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
114 places[c].flags |= TTM_PL_FLAG_WC |
115 TTM_PL_FLAG_UNCACHED;
117 places[c].flags |= TTM_PL_FLAG_CACHED;
121 if (domain & AMDGPU_GEM_DOMAIN_GDS) {
124 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
128 if (domain & AMDGPU_GEM_DOMAIN_GWS) {
131 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
135 if (domain & AMDGPU_GEM_DOMAIN_OA) {
138 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
145 places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
149 placement->num_placement = c;
150 placement->placement = places;
152 placement->num_busy_placement = c;
153 placement->busy_placement = places;
156 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
158 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
160 amdgpu_ttm_placement_init(adev, &abo->placement, abo->placements,
164 static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
165 struct ttm_placement *placement)
167 BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));
169 memcpy(bo->placements, placement->placement,
170 placement->num_placement * sizeof(struct ttm_place));
171 bo->placement.num_placement = placement->num_placement;
172 bo->placement.num_busy_placement = placement->num_busy_placement;
173 bo->placement.placement = bo->placements;
174 bo->placement.busy_placement = bo->placements;
178 * amdgpu_bo_create_reserved - create reserved BO for kernel use
180 * @adev: amdgpu device object
181 * @size: size for the new BO
182 * @align: alignment for the new BO
183 * @domain: where to place it
184 * @bo_ptr: resulting BO
185 * @gpu_addr: GPU addr of the pinned BO
186 * @cpu_addr: optional CPU address mapping
188 * Allocates and pins a BO for kernel internal use, and returns it still
191 * Returns 0 on success, negative error code otherwise.
193 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
194 unsigned long size, int align,
195 u32 domain, struct amdgpu_bo **bo_ptr,
196 u64 *gpu_addr, void **cpu_addr)
202 r = amdgpu_bo_create(adev, size, align, true, domain,
203 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
204 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
205 NULL, NULL, 0, bo_ptr);
207 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
214 r = amdgpu_bo_reserve(*bo_ptr, false);
216 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
220 r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
222 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
223 goto error_unreserve;
227 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
229 dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
230 goto error_unreserve;
237 amdgpu_bo_unreserve(*bo_ptr);
241 amdgpu_bo_unref(bo_ptr);
247 * amdgpu_bo_create_kernel - create BO for kernel use
249 * @adev: amdgpu device object
250 * @size: size for the new BO
251 * @align: alignment for the new BO
252 * @domain: where to place it
253 * @bo_ptr: resulting BO
254 * @gpu_addr: GPU addr of the pinned BO
255 * @cpu_addr: optional CPU address mapping
257 * Allocates and pins a BO for kernel internal use.
259 * Returns 0 on success, negative error code otherwise.
261 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
262 unsigned long size, int align,
263 u32 domain, struct amdgpu_bo **bo_ptr,
264 u64 *gpu_addr, void **cpu_addr)
268 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
274 amdgpu_bo_unreserve(*bo_ptr);
280 * amdgpu_bo_free_kernel - free BO for kernel use
282 * @bo: amdgpu BO to free
284 * unmaps and unpin a BO for kernel internal use.
286 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
292 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
294 amdgpu_bo_kunmap(*bo);
296 amdgpu_bo_unpin(*bo);
297 amdgpu_bo_unreserve(*bo);
308 int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
309 unsigned long size, int byte_align,
310 bool kernel, u32 domain, u64 flags,
312 struct ttm_placement *placement,
313 struct reservation_object *resv,
315 struct amdgpu_bo **bo_ptr)
317 struct amdgpu_bo *bo;
318 enum ttm_bo_type type;
319 unsigned long page_align;
320 u64 initial_bytes_moved, bytes_moved;
324 page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
325 size = ALIGN(size, PAGE_SIZE);
328 type = ttm_bo_type_kernel;
330 type = ttm_bo_type_sg;
332 type = ttm_bo_type_device;
336 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
337 sizeof(struct amdgpu_bo));
339 bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
342 r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
347 INIT_LIST_HEAD(&bo->shadow_list);
348 INIT_LIST_HEAD(&bo->va);
349 bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
350 AMDGPU_GEM_DOMAIN_GTT |
351 AMDGPU_GEM_DOMAIN_CPU |
352 AMDGPU_GEM_DOMAIN_GDS |
353 AMDGPU_GEM_DOMAIN_GWS |
354 AMDGPU_GEM_DOMAIN_OA);
355 bo->allowed_domains = bo->preferred_domains;
356 if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
357 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
362 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
363 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
365 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
366 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
367 /* Don't try to enable write-combining when it can't work, or things
369 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
372 #ifndef CONFIG_COMPILE_TEST
373 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
374 thanks to write-combining
377 if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
378 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
379 "better performance thanks to write-combining\n");
380 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
382 /* For architectures that don't support WC memory,
383 * mask out the WC flag from the BO
385 if (!drm_arch_can_wc_memory())
386 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
389 amdgpu_fill_placement_to_bo(bo, placement);
390 /* Kernel allocation are uninterruptible */
392 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
393 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
394 &bo->placement, page_align, !kernel, NULL,
395 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
396 if (unlikely(r != 0))
399 bytes_moved = atomic64_read(&adev->num_bytes_moved) -
401 if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
402 bo->tbo.mem.mem_type == TTM_PL_VRAM &&
403 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
404 amdgpu_cs_report_moved_bytes(adev, bytes_moved, bytes_moved);
406 amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0);
409 bo->tbo.priority = 1;
411 if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
412 bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
413 struct dma_fence *fence;
415 r = amdgpu_fill_buffer(bo, init_value, bo->tbo.resv, &fence);
419 amdgpu_bo_fence(bo, fence, false);
420 dma_fence_put(bo->tbo.moving);
421 bo->tbo.moving = dma_fence_get(fence);
422 dma_fence_put(fence);
425 amdgpu_bo_unreserve(bo);
428 trace_amdgpu_bo_create(bo);
430 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
431 if (type == ttm_bo_type_device)
432 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
438 ww_mutex_unlock(&bo->tbo.resv->lock);
439 amdgpu_bo_unref(&bo);
443 static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
444 unsigned long size, int byte_align,
445 struct amdgpu_bo *bo)
447 struct ttm_placement placement = {0};
448 struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
454 memset(&placements, 0, sizeof(placements));
455 amdgpu_ttm_placement_init(adev, &placement, placements,
456 AMDGPU_GEM_DOMAIN_GTT,
457 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
458 AMDGPU_GEM_CREATE_SHADOW);
460 r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
461 AMDGPU_GEM_DOMAIN_GTT,
462 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
463 AMDGPU_GEM_CREATE_SHADOW,
469 bo->shadow->parent = amdgpu_bo_ref(bo);
470 mutex_lock(&adev->shadow_list_lock);
471 list_add_tail(&bo->shadow_list, &adev->shadow_list);
472 mutex_unlock(&adev->shadow_list_lock);
478 /* init_value will only take effect when flags contains
479 * AMDGPU_GEM_CREATE_VRAM_CLEARED.
481 int amdgpu_bo_create(struct amdgpu_device *adev,
482 unsigned long size, int byte_align,
483 bool kernel, u32 domain, u64 flags,
485 struct reservation_object *resv,
487 struct amdgpu_bo **bo_ptr)
489 struct ttm_placement placement = {0};
490 struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
491 uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
494 memset(&placements, 0, sizeof(placements));
495 amdgpu_ttm_placement_init(adev, &placement, placements,
496 domain, parent_flags);
498 r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel, domain,
499 parent_flags, sg, &placement, resv,
504 if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {
506 WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
509 r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
512 reservation_object_unlock((*bo_ptr)->tbo.resv);
515 amdgpu_bo_unref(bo_ptr);
521 int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
522 struct amdgpu_ring *ring,
523 struct amdgpu_bo *bo,
524 struct reservation_object *resv,
525 struct dma_fence **fence,
529 struct amdgpu_bo *shadow = bo->shadow;
530 uint64_t bo_addr, shadow_addr;
536 bo_addr = amdgpu_bo_gpu_offset(bo);
537 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
539 r = reservation_object_reserve_shared(bo->tbo.resv);
543 r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
544 amdgpu_bo_size(bo), resv, fence,
547 amdgpu_bo_fence(bo, *fence, true);
553 int amdgpu_bo_validate(struct amdgpu_bo *bo)
561 domain = bo->preferred_domains;
564 amdgpu_ttm_placement_from_domain(bo, domain);
565 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
566 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
567 domain = bo->allowed_domains;
574 int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
575 struct amdgpu_ring *ring,
576 struct amdgpu_bo *bo,
577 struct reservation_object *resv,
578 struct dma_fence **fence,
582 struct amdgpu_bo *shadow = bo->shadow;
583 uint64_t bo_addr, shadow_addr;
589 bo_addr = amdgpu_bo_gpu_offset(bo);
590 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
592 r = reservation_object_reserve_shared(bo->tbo.resv);
596 r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
597 amdgpu_bo_size(bo), resv, fence,
600 amdgpu_bo_fence(bo, *fence, true);
606 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
611 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
614 kptr = amdgpu_bo_kptr(bo);
621 r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
622 MAX_SCHEDULE_TIMEOUT);
626 r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
631 *ptr = amdgpu_bo_kptr(bo);
636 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
640 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
643 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
646 ttm_bo_kunmap(&bo->kmap);
649 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
654 ttm_bo_reference(&bo->tbo);
658 void amdgpu_bo_unref(struct amdgpu_bo **bo)
660 struct ttm_buffer_object *tbo;
671 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
672 u64 min_offset, u64 max_offset,
675 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
679 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
682 if (WARN_ON_ONCE(min_offset > max_offset))
685 /* A shared bo cannot be migrated to VRAM */
686 if (bo->prime_shared_count) {
687 if (domain & AMDGPU_GEM_DOMAIN_GTT)
688 domain = AMDGPU_GEM_DOMAIN_GTT;
694 uint32_t mem_type = bo->tbo.mem.mem_type;
696 if (domain != amdgpu_mem_type_to_domain(mem_type))
701 *gpu_addr = amdgpu_bo_gpu_offset(bo);
703 if (max_offset != 0) {
704 u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
705 WARN_ON_ONCE(max_offset <
706 (amdgpu_bo_gpu_offset(bo) - domain_start));
712 bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
713 amdgpu_ttm_placement_from_domain(bo, domain);
714 for (i = 0; i < bo->placement.num_placement; i++) {
715 /* force to pin into visible video ram */
716 if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
717 !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
718 (!max_offset || max_offset >
719 adev->mc.visible_vram_size)) {
720 if (WARN_ON_ONCE(min_offset >
721 adev->mc.visible_vram_size))
723 fpfn = min_offset >> PAGE_SHIFT;
724 lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
726 fpfn = min_offset >> PAGE_SHIFT;
727 lpfn = max_offset >> PAGE_SHIFT;
729 if (fpfn > bo->placements[i].fpfn)
730 bo->placements[i].fpfn = fpfn;
731 if (!bo->placements[i].lpfn ||
732 (lpfn && lpfn < bo->placements[i].lpfn))
733 bo->placements[i].lpfn = lpfn;
734 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
737 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
739 dev_err(adev->dev, "%p pin failed\n", bo);
744 if (gpu_addr != NULL) {
745 r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
747 dev_err(adev->dev, "%p bind failed\n", bo);
750 *gpu_addr = amdgpu_bo_gpu_offset(bo);
752 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
753 adev->vram_pin_size += amdgpu_bo_size(bo);
754 adev->invisible_pin_size += amdgpu_vram_mgr_bo_invisible_size(bo);
755 } else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
756 adev->gart_pin_size += amdgpu_bo_size(bo);
763 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
765 return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
768 int amdgpu_bo_unpin(struct amdgpu_bo *bo)
770 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
773 if (!bo->pin_count) {
774 dev_warn(adev->dev, "%p unpin not necessary\n", bo);
780 for (i = 0; i < bo->placement.num_placement; i++) {
781 bo->placements[i].lpfn = 0;
782 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
784 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
786 dev_err(adev->dev, "%p validate failed for unpin\n", bo);
790 if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
791 adev->vram_pin_size -= amdgpu_bo_size(bo);
792 adev->invisible_pin_size -= amdgpu_vram_mgr_bo_invisible_size(bo);
793 } else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
794 adev->gart_pin_size -= amdgpu_bo_size(bo);
801 int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
803 /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
804 if (0 && (adev->flags & AMD_IS_APU)) {
805 /* Useless to evict on IGP chips */
808 return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
811 static const char *amdgpu_vram_names[] = {
822 int amdgpu_bo_init(struct amdgpu_device *adev)
824 /* reserve PAT memory space to WC for VRAM */
825 arch_io_reserve_memtype_wc(adev->mc.aper_base,
828 /* Add an MTRR for the VRAM */
829 adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
831 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
832 adev->mc.mc_vram_size >> 20,
833 (unsigned long long)adev->mc.aper_size >> 20);
834 DRM_INFO("RAM width %dbits %s\n",
835 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
836 return amdgpu_ttm_init(adev);
839 void amdgpu_bo_fini(struct amdgpu_device *adev)
841 amdgpu_ttm_fini(adev);
842 arch_phys_wc_del(adev->mc.vram_mtrr);
843 arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size);
846 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
847 struct vm_area_struct *vma)
849 return ttm_fbdev_mmap(vma, &bo->tbo);
852 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
854 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
856 if (adev->family <= AMDGPU_FAMILY_CZ &&
857 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
860 bo->tiling_flags = tiling_flags;
864 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
866 lockdep_assert_held(&bo->tbo.resv->lock.base);
869 *tiling_flags = bo->tiling_flags;
872 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
873 uint32_t metadata_size, uint64_t flags)
877 if (!metadata_size) {
878 if (bo->metadata_size) {
881 bo->metadata_size = 0;
886 if (metadata == NULL)
889 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
894 bo->metadata_flags = flags;
895 bo->metadata = buffer;
896 bo->metadata_size = metadata_size;
901 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
902 size_t buffer_size, uint32_t *metadata_size,
905 if (!buffer && !metadata_size)
909 if (buffer_size < bo->metadata_size)
912 if (bo->metadata_size)
913 memcpy(buffer, bo->metadata, bo->metadata_size);
917 *metadata_size = bo->metadata_size;
919 *flags = bo->metadata_flags;
924 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
926 struct ttm_mem_reg *new_mem)
928 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
929 struct amdgpu_bo *abo;
930 struct ttm_mem_reg *old_mem = &bo->mem;
932 if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
935 abo = container_of(bo, struct amdgpu_bo, tbo);
936 amdgpu_vm_bo_invalidate(adev, abo);
938 amdgpu_bo_kunmap(abo);
940 /* remember the eviction */
942 atomic64_inc(&adev->num_evictions);
944 /* update statistics */
948 /* move_notify is called before move happens */
949 trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
952 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
954 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
955 struct amdgpu_bo *abo;
956 unsigned long offset, size;
959 if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
962 abo = container_of(bo, struct amdgpu_bo, tbo);
964 /* Remember that this BO was accessed by the CPU */
965 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
967 if (bo->mem.mem_type != TTM_PL_VRAM)
970 size = bo->mem.num_pages << PAGE_SHIFT;
971 offset = bo->mem.start << PAGE_SHIFT;
972 if ((offset + size) <= adev->mc.visible_vram_size)
975 /* Can't move a pinned BO to visible VRAM */
976 if (abo->pin_count > 0)
979 /* hurrah the memory is not visible ! */
980 atomic64_inc(&adev->num_vram_cpu_page_faults);
981 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
982 AMDGPU_GEM_DOMAIN_GTT);
984 /* Avoid costly evictions; only set GTT as a busy placement */
985 abo->placement.num_busy_placement = 1;
986 abo->placement.busy_placement = &abo->placements[1];
988 r = ttm_bo_validate(bo, &abo->placement, false, false);
989 if (unlikely(r != 0))
992 offset = bo->mem.start << PAGE_SHIFT;
993 /* this should never happen */
994 if (bo->mem.mem_type == TTM_PL_VRAM &&
995 (offset + size) > adev->mc.visible_vram_size)
1002 * amdgpu_bo_fence - add fence to buffer object
1004 * @bo: buffer object in question
1005 * @fence: fence to add
1006 * @shared: true if fence should be added shared
1009 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1012 struct reservation_object *resv = bo->tbo.resv;
1015 reservation_object_add_shared_fence(resv, fence);
1017 reservation_object_add_excl_fence(resv, fence);
1021 * amdgpu_bo_gpu_offset - return GPU offset of bo
1022 * @bo: amdgpu object for which we query the offset
1024 * Returns current GPU offset of the object.
1026 * Note: object should either be pinned or reserved when calling this
1027 * function, it might be useful to add check for this for debugging.
1029 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1031 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
1032 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
1033 !amdgpu_ttm_is_bound(bo->tbo.ttm));
1034 WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
1036 WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
1037 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
1038 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1040 return bo->tbo.offset;