drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #include <drm/ttm/ttm_bo_driver.h>
  33 #include <drm/ttm/ttm_placement.h>
  34 #include <drm/drm_vma_manager.h>
  35 #include <linux/io.h>
  36 #include <linux/highmem.h>
  37 #include <linux/wait.h>
  38 #include <linux/slab.h>
  39 #include <linux/vmalloc.h>
  40 #include <linux/module.h>
  41 #include <linux/dma-resv.h>
  42
  43 struct ttm_transfer_obj {
  44         struct ttm_buffer_object base;
  45         struct ttm_buffer_object *bo;
  46 };
  47
  48 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  49 {
  50         ttm_bo_mem_put(bo, &bo->mem);
  51 }
  52
  53 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  54                    struct ttm_operation_ctx *ctx,
  55                     struct ttm_mem_reg *new_mem)
  56 {
  57         struct ttm_tt *ttm = bo->ttm;
  58         struct ttm_mem_reg *old_mem = &bo->mem;
  59         int ret;
  60
  61         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  62                 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
  63
  64                 if (unlikely(ret != 0)) {
  65                         if (ret != -ERESTARTSYS)
  66                                 pr_err("Failed to expire sync object before unbinding TTM\n");
  67                         return ret;
  68                 }
  69
  70                 ttm_tt_unbind(ttm);
  71                 ttm_bo_free_old_node(bo);
  72                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  73                                 TTM_PL_MASK_MEM);
  74                 old_mem->mem_type = TTM_PL_SYSTEM;
  75         }
  76
  77         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  78         if (unlikely(ret != 0))
  79                 return ret;
  80
  81         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  82                 ret = ttm_tt_bind(ttm, new_mem, ctx);
  83                 if (unlikely(ret != 0))
  84                         return ret;
  85         }
  86
  87         *old_mem = *new_mem;
  88         new_mem->mm_node = NULL;
  89
  90         return 0;
  91 }
  92 EXPORT_SYMBOL(ttm_bo_move_ttm);
  93
  94 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
  95 {
  96         if (likely(man->io_reserve_fastpath))
  97                 return 0;
  98
  99         if (interruptible)
 100                 return mutex_lock_interruptible(&man->io_reserve_mutex);
 101
 102         mutex_lock(&man->io_reserve_mutex);
 103         return 0;
 104 }
 105 EXPORT_SYMBOL(ttm_mem_io_lock);
 106
 107 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
 108 {
 109         if (likely(man->io_reserve_fastpath))
 110                 return;
 111
 112         mutex_unlock(&man->io_reserve_mutex);
 113 }
 114 EXPORT_SYMBOL(ttm_mem_io_unlock);
 115
 116 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
 117 {
 118         struct ttm_buffer_object *bo;
 119
 120         if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
 121                 return -EAGAIN;
 122
 123         bo = list_first_entry(&man->io_reserve_lru,
 124                               struct ttm_buffer_object,
 125                               io_reserve_lru);
 126         list_del_init(&bo->io_reserve_lru);
 127         ttm_bo_unmap_virtual_locked(bo);
 128
 129         return 0;
 130 }
 131
 132
 133 int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
 134                        struct ttm_mem_reg *mem)
 135 {
 136         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 137         int ret = 0;
 138
 139         if (!bdev->driver->io_mem_reserve)
 140                 return 0;
 141         if (likely(man->io_reserve_fastpath))
 142                 return bdev->driver->io_mem_reserve(bdev, mem);
 143
 144         if (bdev->driver->io_mem_reserve &&
 145             mem->bus.io_reserved_count++ == 0) {
 146 retry:
 147                 ret = bdev->driver->io_mem_reserve(bdev, mem);
 148                 if (ret == -EAGAIN) {
 149                         ret = ttm_mem_io_evict(man);
 150                         if (ret == 0)
 151                                 goto retry;
 152                 }
 153         }
 154         return ret;
 155 }
 156 EXPORT_SYMBOL(ttm_mem_io_reserve);
 157
 158 void ttm_mem_io_free(struct ttm_bo_device *bdev,
 159                      struct ttm_mem_reg *mem)
 160 {
 161         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 162
 163         if (likely(man->io_reserve_fastpath))
 164                 return;
 165
 166         if (bdev->driver->io_mem_reserve &&
 167             --mem->bus.io_reserved_count == 0 &&
 168             bdev->driver->io_mem_free)
 169                 bdev->driver->io_mem_free(bdev, mem);
 170
 171 }
 172 EXPORT_SYMBOL(ttm_mem_io_free);
 173
 174 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
 175 {
 176         struct ttm_mem_reg *mem = &bo->mem;
 177         int ret;
 178
 179         if (!mem->bus.io_reserved_vm) {
 180                 struct ttm_mem_type_manager *man =
 181                         &bo->bdev->man[mem->mem_type];
 182
 183                 ret = ttm_mem_io_reserve(bo->bdev, mem);
 184                 if (unlikely(ret != 0))
 185                         return ret;
 186                 mem->bus.io_reserved_vm = true;
 187                 if (man->use_io_reserve_lru)
 188                         list_add_tail(&bo->io_reserve_lru,
 189                                       &man->io_reserve_lru);
 190         }
 191         return 0;
 192 }
 193
 194 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 195 {
 196         struct ttm_mem_reg *mem = &bo->mem;
 197
 198         if (mem->bus.io_reserved_vm) {
 199                 mem->bus.io_reserved_vm = false;
 200                 list_del_init(&bo->io_reserve_lru);
 201                 ttm_mem_io_free(bo->bdev, mem);
 202         }
 203 }
 204
 205 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 206                         void **virtual)
 207 {
 208         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 209         int ret;
 210         void *addr;
 211
 212         *virtual = NULL;
 213         (void) ttm_mem_io_lock(man, false);
 214         ret = ttm_mem_io_reserve(bdev, mem);
 215         ttm_mem_io_unlock(man);
 216         if (ret || !mem->bus.is_iomem)
 217                 return ret;
 218
 219         if (mem->bus.addr) {
 220                 addr = mem->bus.addr;
 221         } else {
 222                 if (mem->placement & TTM_PL_FLAG_WC)
 223                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 224                 else
 225                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 226                 if (!addr) {
 227                         (void) ttm_mem_io_lock(man, false);
 228                         ttm_mem_io_free(bdev, mem);
 229                         ttm_mem_io_unlock(man);
 230                         return -ENOMEM;
 231                 }
 232         }
 233         *virtual = addr;
 234         return 0;
 235 }
 236
 237 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 238                          void *virtual)
 239 {
 240         struct ttm_mem_type_manager *man;
 241
 242         man = &bdev->man[mem->mem_type];
 243
 244         if (virtual && mem->bus.addr == NULL)
 245                 iounmap(virtual);
 246         (void) ttm_mem_io_lock(man, false);
 247         ttm_mem_io_free(bdev, mem);
 248         ttm_mem_io_unlock(man);
 249 }
 250
 251 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 252 {
 253         uint32_t *dstP =
 254             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 255         uint32_t *srcP =
 256             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 257
 258         int i;
 259         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 260                 iowrite32(ioread32(srcP++), dstP++);
 261         return 0;
 262 }
 263
 264 #ifdef CONFIG_X86
 265 #define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot)
 266 #define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr)
 267 #else
 268 #define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0,  __prot)
 269 #define __ttm_kunmap_atomic(__addr) vunmap(__addr)
 270 #endif
 271
 272
 273 /**
 274  * ttm_kmap_atomic_prot - Efficient kernel map of a single page with
 275  * specified page protection.
 276  *
 277  * @page: The page to map.
 278  * @prot: The page protection.
 279  *
 280  * This function maps a TTM page using the kmap_atomic api if available,
 281  * otherwise falls back to vmap. The user must make sure that the
 282  * specified page does not have an aliased mapping with a different caching
 283  * policy unless the architecture explicitly allows it. Also mapping and
 284  * unmapping using this api must be correctly nested. Unmapping should
 285  * occur in the reverse order of mapping.
 286  */
 287 void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot)
 288 {
 289         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
 290                 return kmap_atomic(page);
 291         else
 292                 return __ttm_kmap_atomic_prot(page, prot);
 293 }
 294 EXPORT_SYMBOL(ttm_kmap_atomic_prot);
 295
 296 /**
 297  * ttm_kunmap_atomic_prot - Unmap a page that was mapped using
 298  * ttm_kmap_atomic_prot.
 299  *
 300  * @addr: The virtual address from the map.
 301  * @prot: The page protection.
 302  */
 303 void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot)
 304 {
 305         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
 306                 kunmap_atomic(addr);
 307         else
 308                 __ttm_kunmap_atomic(addr);
 309 }
 310 EXPORT_SYMBOL(ttm_kunmap_atomic_prot);
 311
 312 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 313                                 unsigned long page,
 314                                 pgprot_t prot)
 315 {
 316         struct page *d = ttm->pages[page];
 317         void *dst;
 318
 319         if (!d)
 320                 return -ENOMEM;
 321
 322         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 323         dst = ttm_kmap_atomic_prot(d, prot);
 324         if (!dst)
 325                 return -ENOMEM;
 326
 327         memcpy_fromio(dst, src, PAGE_SIZE);
 328
 329         ttm_kunmap_atomic_prot(dst, prot);
 330
 331         return 0;
 332 }
 333
 334 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 335                                 unsigned long page,
 336                                 pgprot_t prot)
 337 {
 338         struct page *s = ttm->pages[page];
 339         void *src;
 340
 341         if (!s)
 342                 return -ENOMEM;
 343
 344         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 345         src = ttm_kmap_atomic_prot(s, prot);
 346         if (!src)
 347                 return -ENOMEM;
 348
 349         memcpy_toio(dst, src, PAGE_SIZE);
 350
 351         ttm_kunmap_atomic_prot(src, prot);
 352
 353         return 0;
 354 }
 355
 356 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 357                        struct ttm_operation_ctx *ctx,
 358                        struct ttm_mem_reg *new_mem)
 359 {
 360         struct ttm_bo_device *bdev = bo->bdev;
 361         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 362         struct ttm_tt *ttm = bo->ttm;
 363         struct ttm_mem_reg *old_mem = &bo->mem;
 364         struct ttm_mem_reg old_copy = *old_mem;
 365         void *old_iomap;
 366         void *new_iomap;
 367         int ret;
 368         unsigned long i;
 369         unsigned long page;
 370         unsigned long add = 0;
 371         int dir;
 372
 373         ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 374         if (ret)
 375                 return ret;
 376
 377         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 378         if (ret)
 379                 return ret;
 380         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 381         if (ret)
 382                 goto out;
 383
 384         /*
 385          * Single TTM move. NOP.
 386          */
 387         if (old_iomap == NULL && new_iomap == NULL)
 388                 goto out2;
 389
 390         /*
 391          * Don't move nonexistent data. Clear destination instead.
 392          */
 393         if (old_iomap == NULL &&
 394             (ttm == NULL || (ttm->state == tt_unpopulated &&
 395                              !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
 396                 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
 397                 goto out2;
 398         }
 399
 400         /*
 401          * TTM might be null for moves within the same region.
 402          */
 403         if (ttm) {
 404                 ret = ttm_tt_populate(ttm, ctx);
 405                 if (ret)
 406                         goto out1;
 407         }
 408
 409         add = 0;
 410         dir = 1;
 411
 412         if ((old_mem->mem_type == new_mem->mem_type) &&
 413             (new_mem->start < old_mem->start + old_mem->size)) {
 414                 dir = -1;
 415                 add = new_mem->num_pages - 1;
 416         }
 417
 418         for (i = 0; i < new_mem->num_pages; ++i) {
 419                 page = i * dir + add;
 420                 if (old_iomap == NULL) {
 421                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 422                                                     PAGE_KERNEL);
 423                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 424                                                    prot);
 425                 } else if (new_iomap == NULL) {
 426                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 427                                                     PAGE_KERNEL);
 428                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 429                                                    prot);
 430                 } else {
 431                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 432                 }
 433                 if (ret)
 434                         goto out1;
 435         }
 436         mb();
 437 out2:
 438         old_copy = *old_mem;
 439         *old_mem = *new_mem;
 440         new_mem->mm_node = NULL;
 441
 442         if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 443                 ttm_tt_destroy(ttm);
 444                 bo->ttm = NULL;
 445         }
 446
 447 out1:
 448         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 449 out:
 450         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 451
 452         /*
 453          * On error, keep the mm node!
 454          */
 455         if (!ret)
 456                 ttm_bo_mem_put(bo, &old_copy);
 457         return ret;
 458 }
 459 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 460
 461 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 462 {
 463         struct ttm_transfer_obj *fbo;
 464
 465         fbo = container_of(bo, struct ttm_transfer_obj, base);
 466         ttm_bo_put(fbo->bo);
 467         kfree(fbo);
 468 }
 469
 470 /**
 471  * ttm_buffer_object_transfer
 472  *
 473  * @bo: A pointer to a struct ttm_buffer_object.
 474  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 475  * holding the data of @bo with the old placement.
 476  *
 477  * This is a utility function that may be called after an accelerated move
 478  * has been scheduled. A new buffer object is created as a placeholder for
 479  * the old data while it's being copied. When that buffer object is idle,
 480  * it can be destroyed, releasing the space of the old placement.
 481  * Returns:
 482  * !0: Failure.
 483  */
 484
 485 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 486                                       struct ttm_buffer_object **new_obj)
 487 {
 488         struct ttm_transfer_obj *fbo;
 489         int ret;
 490
 491         fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
 492         if (!fbo)
 493                 return -ENOMEM;
 494
 495         fbo->base = *bo;
 496         fbo->base.mem.placement |= TTM_PL_FLAG_NO_EVICT;
 497
 498         ttm_bo_get(bo);
 499         fbo->bo = bo;
 500
 501         /**
 502          * Fix up members that we shouldn't copy directly:
 503          * TODO: Explicit member copy would probably be better here.
 504          */
 505
 506         atomic_inc(&bo->bdev->glob->bo_count);
 507         INIT_LIST_HEAD(&fbo->base.ddestroy);
 508         INIT_LIST_HEAD(&fbo->base.lru);
 509         INIT_LIST_HEAD(&fbo->base.swap);
 510         INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
 511         mutex_init(&fbo->base.wu_mutex);
 512         fbo->base.moving = NULL;
 513         drm_vma_node_reset(&fbo->base.base.vma_node);
 514         atomic_set(&fbo->base.cpu_writers, 0);
 515
 516         kref_init(&fbo->base.list_kref);
 517         kref_init(&fbo->base.kref);
 518         fbo->base.destroy = &ttm_transfered_destroy;
 519         fbo->base.acc_size = 0;
 520         fbo->base.base.resv = &fbo->base.base._resv;
 521         dma_resv_init(fbo->base.base.resv);
 522         ret = dma_resv_trylock(fbo->base.base.resv);
 523         WARN_ON(!ret);
 524
 525         *new_obj = &fbo->base;
 526         return 0;
 527 }
 528
 529 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 530 {
 531         /* Cached mappings need no adjustment */
 532         if (caching_flags & TTM_PL_FLAG_CACHED)
 533                 return tmp;
 534
 535 #if defined(__i386__) || defined(__x86_64__)
 536         if (caching_flags & TTM_PL_FLAG_WC)
 537                 tmp = pgprot_writecombine(tmp);
 538         else if (boot_cpu_data.x86 > 3)
 539                 tmp = pgprot_noncached(tmp);
 540 #endif
 541 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
 542     defined(__powerpc__) || defined(__mips__)
 543         if (caching_flags & TTM_PL_FLAG_WC)
 544                 tmp = pgprot_writecombine(tmp);
 545         else
 546                 tmp = pgprot_noncached(tmp);
 547 #endif
 548 #if defined(__sparc__)
 549         tmp = pgprot_noncached(tmp);
 550 #endif
 551         return tmp;
 552 }
 553 EXPORT_SYMBOL(ttm_io_prot);
 554
 555 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 556                           unsigned long offset,
 557                           unsigned long size,
 558                           struct ttm_bo_kmap_obj *map)
 559 {
 560         struct ttm_mem_reg *mem = &bo->mem;
 561
 562         if (bo->mem.bus.addr) {
 563                 map->bo_kmap_type = ttm_bo_map_premapped;
 564                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 565         } else {
 566                 map->bo_kmap_type = ttm_bo_map_iomap;
 567                 if (mem->placement & TTM_PL_FLAG_WC)
 568                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 569                                                   size);
 570                 else
 571                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 572                                                        size);
 573         }
 574         return (!map->virtual) ? -ENOMEM : 0;
 575 }
 576
 577 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 578                            unsigned long start_page,
 579                            unsigned long num_pages,
 580                            struct ttm_bo_kmap_obj *map)
 581 {
 582         struct ttm_mem_reg *mem = &bo->mem;
 583         struct ttm_operation_ctx ctx = {
 584                 .interruptible = false,
 585                 .no_wait_gpu = false
 586         };
 587         struct ttm_tt *ttm = bo->ttm;
 588         pgprot_t prot;
 589         int ret;
 590
 591         BUG_ON(!ttm);
 592
 593         ret = ttm_tt_populate(ttm, &ctx);
 594         if (ret)
 595                 return ret;
 596
 597         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 598                 /*
 599                  * We're mapping a single page, and the desired
 600                  * page protection is consistent with the bo.
 601                  */
 602
 603                 map->bo_kmap_type = ttm_bo_map_kmap;
 604                 map->page = ttm->pages[start_page];
 605                 map->virtual = kmap(map->page);
 606         } else {
 607                 /*
 608                  * We need to use vmap to get the desired page protection
 609                  * or to make the buffer object look contiguous.
 610                  */
 611                 prot = ttm_io_prot(mem->placement, PAGE_KERNEL);
 612                 map->bo_kmap_type = ttm_bo_map_vmap;
 613                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 614                                     0, prot);
 615         }
 616         return (!map->virtual) ? -ENOMEM : 0;
 617 }
 618
 619 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 620                 unsigned long start_page, unsigned long num_pages,
 621                 struct ttm_bo_kmap_obj *map)
 622 {
 623         struct ttm_mem_type_manager *man =
 624                 &bo->bdev->man[bo->mem.mem_type];
 625         unsigned long offset, size;
 626         int ret;
 627
 628         map->virtual = NULL;
 629         map->bo = bo;
 630         if (num_pages > bo->num_pages)
 631                 return -EINVAL;
 632         if (start_page > bo->num_pages)
 633                 return -EINVAL;
 634
 635         (void) ttm_mem_io_lock(man, false);
 636         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 637         ttm_mem_io_unlock(man);
 638         if (ret)
 639                 return ret;
 640         if (!bo->mem.bus.is_iomem) {
 641                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 642         } else {
 643                 offset = start_page << PAGE_SHIFT;
 644                 size = num_pages << PAGE_SHIFT;
 645                 return ttm_bo_ioremap(bo, offset, size, map);
 646         }
 647 }
 648 EXPORT_SYMBOL(ttm_bo_kmap);
 649
 650 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 651 {
 652         struct ttm_buffer_object *bo = map->bo;
 653         struct ttm_mem_type_manager *man =
 654                 &bo->bdev->man[bo->mem.mem_type];
 655
 656         if (!map->virtual)
 657                 return;
 658         switch (map->bo_kmap_type) {
 659         case ttm_bo_map_iomap:
 660                 iounmap(map->virtual);
 661                 break;
 662         case ttm_bo_map_vmap:
 663                 vunmap(map->virtual);
 664                 break;
 665         case ttm_bo_map_kmap:
 666                 kunmap(map->page);
 667                 break;
 668         case ttm_bo_map_premapped:
 669                 break;
 670         default:
 671                 BUG();
 672         }
 673         (void) ttm_mem_io_lock(man, false);
 674         ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 675         ttm_mem_io_unlock(man);
 676         map->virtual = NULL;
 677         map->page = NULL;
 678 }
 679 EXPORT_SYMBOL(ttm_bo_kunmap);
 680
 681 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 682                               struct dma_fence *fence,
 683                               bool evict,
 684                               struct ttm_mem_reg *new_mem)
 685 {
 686         struct ttm_bo_device *bdev = bo->bdev;
 687         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 688         struct ttm_mem_reg *old_mem = &bo->mem;
 689         int ret;
 690         struct ttm_buffer_object *ghost_obj;
 691
 692         dma_resv_add_excl_fence(bo->base.resv, fence);
 693         if (evict) {
 694                 ret = ttm_bo_wait(bo, false, false);
 695                 if (ret)
 696                         return ret;
 697
 698                 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 699                         ttm_tt_destroy(bo->ttm);
 700                         bo->ttm = NULL;
 701                 }
 702                 ttm_bo_free_old_node(bo);
 703         } else {
 704                 /**
 705                  * This should help pipeline ordinary buffer moves.
 706                  *
 707                  * Hang old buffer memory on a new buffer object,
 708                  * and leave it to be released when the GPU
 709                  * operation has completed.
 710                  */
 711
 712                 dma_fence_put(bo->moving);
 713                 bo->moving = dma_fence_get(fence);
 714
 715                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 716                 if (ret)
 717                         return ret;
 718
 719                 dma_resv_add_excl_fence(ghost_obj->base.resv, fence);
 720
 721                 /**
 722                  * If we're not moving to fixed memory, the TTM object
 723                  * needs to stay alive. Otherwhise hang it on the ghost
 724                  * bo to be unbound and destroyed.
 725                  */
 726
 727                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 728                         ghost_obj->ttm = NULL;
 729                 else
 730                         bo->ttm = NULL;
 731
 732                 ttm_bo_unreserve(ghost_obj);
 733                 ttm_bo_put(ghost_obj);
 734         }
 735
 736         *old_mem = *new_mem;
 737         new_mem->mm_node = NULL;
 738
 739         return 0;
 740 }
 741 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
 742
 743 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
 744                          struct dma_fence *fence, bool evict,
 745                          struct ttm_mem_reg *new_mem)
 746 {
 747         struct ttm_bo_device *bdev = bo->bdev;
 748         struct ttm_mem_reg *old_mem = &bo->mem;
 749
 750         struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
 751         struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
 752
 753         int ret;
 754
 755         dma_resv_add_excl_fence(bo->base.resv, fence);
 756
 757         if (!evict) {
 758                 struct ttm_buffer_object *ghost_obj;
 759
 760                 /**
 761                  * This should help pipeline ordinary buffer moves.
 762                  *
 763                  * Hang old buffer memory on a new buffer object,
 764                  * and leave it to be released when the GPU
 765                  * operation has completed.
 766                  */
 767
 768                 dma_fence_put(bo->moving);
 769                 bo->moving = dma_fence_get(fence);
 770
 771                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 772                 if (ret)
 773                         return ret;
 774
 775                 dma_resv_add_excl_fence(ghost_obj->base.resv, fence);
 776
 777                 /**
 778                  * If we're not moving to fixed memory, the TTM object
 779                  * needs to stay alive. Otherwhise hang it on the ghost
 780                  * bo to be unbound and destroyed.
 781                  */
 782
 783                 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
 784                         ghost_obj->ttm = NULL;
 785                 else
 786                         bo->ttm = NULL;
 787
 788                 ttm_bo_unreserve(ghost_obj);
 789                 ttm_bo_put(ghost_obj);
 790
 791         } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
 792
 793                 /**
 794                  * BO doesn't have a TTM we need to bind/unbind. Just remember
 795                  * this eviction and free up the allocation
 796                  */
 797
 798                 spin_lock(&from->move_lock);
 799                 if (!from->move || dma_fence_is_later(fence, from->move)) {
 800                         dma_fence_put(from->move);
 801                         from->move = dma_fence_get(fence);
 802                 }
 803                 spin_unlock(&from->move_lock);
 804
 805                 ttm_bo_free_old_node(bo);
 806
 807                 dma_fence_put(bo->moving);
 808                 bo->moving = dma_fence_get(fence);
 809
 810         } else {
 811                 /**
 812                  * Last resort, wait for the move to be completed.
 813                  *
 814                  * Should never happen in pratice.
 815                  */
 816
 817                 ret = ttm_bo_wait(bo, false, false);
 818                 if (ret)
 819                         return ret;
 820
 821                 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
 822                         ttm_tt_destroy(bo->ttm);
 823                         bo->ttm = NULL;
 824                 }
 825                 ttm_bo_free_old_node(bo);
 826         }
 827
 828         *old_mem = *new_mem;
 829         new_mem->mm_node = NULL;
 830
 831         return 0;
 832 }
 833 EXPORT_SYMBOL(ttm_bo_pipeline_move);
 834
 835 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
 836 {
 837         struct ttm_buffer_object *ghost;
 838         int ret;
 839
 840         ret = ttm_buffer_object_transfer(bo, &ghost);
 841         if (ret)
 842                 return ret;
 843
 844         ret = dma_resv_copy_fences(ghost->base.resv, bo->base.resv);
 845         /* Last resort, wait for the BO to be idle when we are OOM */
 846         if (ret)
 847                 ttm_bo_wait(bo, false, false);
 848
 849         memset(&bo->mem, 0, sizeof(bo->mem));
 850         bo->mem.mem_type = TTM_PL_SYSTEM;
 851         bo->ttm = NULL;
 852
 853         ttm_bo_unreserve(ghost);
 854         ttm_bo_put(ghost);
 855
 856         return 0;
 857 }