drivers/misc/mic/scif/scif_dma.c

   1 /*
   2  * Intel MIC Platform Software Stack (MPSS)
   3  *
   4  * Copyright(c) 2015 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License, version 2, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13  * General Public License for more details.
  14  *
  15  * Intel SCIF driver.
  16  *
  17  */
  18 #include "scif_main.h"
  19 #include "scif_map.h"
  20
  21 /*
  22  * struct scif_dma_comp_cb - SCIF DMA completion callback
  23  *
  24  * @dma_completion_func: DMA completion callback
  25  * @cb_cookie: DMA completion callback cookie
  26  * @temp_buf: Temporary buffer
  27  * @temp_buf_to_free: Temporary buffer to be freed
  28  * @is_cache: Is a kmem_cache allocated buffer
  29  * @dst_offset: Destination registration offset
  30  * @dst_window: Destination registration window
  31  * @len: Length of the temp buffer
  32  * @temp_phys: DMA address of the temp buffer
  33  * @sdev: The SCIF device
  34  * @header_padding: padding for cache line alignment
  35  */
  36 struct scif_dma_comp_cb {
  37         void (*dma_completion_func)(void *cookie);
  38         void *cb_cookie;
  39         u8 *temp_buf;
  40         u8 *temp_buf_to_free;
  41         bool is_cache;
  42         s64 dst_offset;
  43         struct scif_window *dst_window;
  44         size_t len;
  45         dma_addr_t temp_phys;
  46         struct scif_dev *sdev;
  47         int header_padding;
  48 };
  49
  50 /**
  51  * struct scif_copy_work - Work for DMA copy
  52  *
  53  * @src_offset: Starting source offset
  54  * @dst_offset: Starting destination offset
  55  * @src_window: Starting src registered window
  56  * @dst_window: Starting dst registered window
  57  * @loopback: true if this is a loopback DMA transfer
  58  * @len: Length of the transfer
  59  * @comp_cb: DMA copy completion callback
  60  * @remote_dev: The remote SCIF peer device
  61  * @fence_type: polling or interrupt based
  62  * @ordered: is this a tail byte ordered DMA transfer
  63  */
  64 struct scif_copy_work {
  65         s64 src_offset;
  66         s64 dst_offset;
  67         struct scif_window *src_window;
  68         struct scif_window *dst_window;
  69         int loopback;
  70         size_t len;
  71         struct scif_dma_comp_cb   *comp_cb;
  72         struct scif_dev *remote_dev;
  73         int fence_type;
  74         bool ordered;
  75 };
  76
  77 /**
  78  * scif_reserve_dma_chan:
  79  * @ep: Endpoint Descriptor.
  80  *
  81  * This routine reserves a DMA channel for a particular
  82  * endpoint. All DMA transfers for an endpoint are always
  83  * programmed on the same DMA channel.
  84  */
  85 int scif_reserve_dma_chan(struct scif_endpt *ep)
  86 {
  87         int err = 0;
  88         struct scif_dev *scifdev;
  89         struct scif_hw_dev *sdev;
  90         struct dma_chan *chan;
  91
  92         /* Loopback DMAs are not supported on the management node */
  93         if (!scif_info.nodeid && scifdev_self(ep->remote_dev))
  94                 return 0;
  95         if (scif_info.nodeid)
  96                 scifdev = &scif_dev[0];
  97         else
  98                 scifdev = ep->remote_dev;
  99         sdev = scifdev->sdev;
 100         if (!sdev->num_dma_ch)
 101                 return -ENODEV;
 102         chan = sdev->dma_ch[scifdev->dma_ch_idx];
 103         scifdev->dma_ch_idx = (scifdev->dma_ch_idx + 1) % sdev->num_dma_ch;
 104         mutex_lock(&ep->rma_info.rma_lock);
 105         ep->rma_info.dma_chan = chan;
 106         mutex_unlock(&ep->rma_info.rma_lock);
 107         return err;
 108 }
 109
 110 #ifdef CONFIG_MMU_NOTIFIER
 111 /**
 112  * scif_rma_destroy_tcw:
 113  *
 114  * This routine destroys temporary cached windows
 115  */
 116 static
 117 void __scif_rma_destroy_tcw(struct scif_mmu_notif *mmn,
 118                             u64 start, u64 len)
 119 {
 120         struct list_head *item, *tmp;
 121         struct scif_window *window;
 122         u64 start_va, end_va;
 123         u64 end = start + len;
 124
 125         if (end <= start)
 126                 return;
 127
 128         list_for_each_safe(item, tmp, &mmn->tc_reg_list) {
 129                 window = list_entry(item, struct scif_window, list);
 130                 if (!len)
 131                         break;
 132                 start_va = window->va_for_temp;
 133                 end_va = start_va + (window->nr_pages << PAGE_SHIFT);
 134                 if (start < start_va && end <= start_va)
 135                         break;
 136                 if (start >= end_va)
 137                         continue;
 138                 __scif_rma_destroy_tcw_helper(window);
 139         }
 140 }
 141
 142 static void scif_rma_destroy_tcw(struct scif_mmu_notif *mmn, u64 start, u64 len)
 143 {
 144         struct scif_endpt *ep = mmn->ep;
 145
 146         spin_lock(&ep->rma_info.tc_lock);
 147         __scif_rma_destroy_tcw(mmn, start, len);
 148         spin_unlock(&ep->rma_info.tc_lock);
 149 }
 150
 151 static void scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
 152 {
 153         struct list_head *item, *tmp;
 154         struct scif_mmu_notif *mmn;
 155
 156         list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
 157                 mmn = list_entry(item, struct scif_mmu_notif, list);
 158                 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
 159         }
 160 }
 161
 162 static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
 163 {
 164         struct list_head *item, *tmp;
 165         struct scif_mmu_notif *mmn;
 166
 167         spin_lock(&ep->rma_info.tc_lock);
 168         list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
 169                 mmn = list_entry(item, struct scif_mmu_notif, list);
 170                 __scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
 171         }
 172         spin_unlock(&ep->rma_info.tc_lock);
 173 }
 174
 175 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
 176 {
 177         if ((cur_bytes >> PAGE_SHIFT) > scif_info.rma_tc_limit)
 178                 return false;
 179         if ((atomic_read(&ep->rma_info.tcw_total_pages)
 180                         + (cur_bytes >> PAGE_SHIFT)) >
 181                         scif_info.rma_tc_limit) {
 182                 dev_info(scif_info.mdev.this_device,
 183                          "%s %d total=%d, current=%zu reached max\n",
 184                          __func__, __LINE__,
 185                          atomic_read(&ep->rma_info.tcw_total_pages),
 186                          (1 + (cur_bytes >> PAGE_SHIFT)));
 187                 scif_rma_destroy_tcw_invalid();
 188                 __scif_rma_destroy_tcw_ep(ep);
 189         }
 190         return true;
 191 }
 192
 193 static void scif_mmu_notifier_release(struct mmu_notifier *mn,
 194                                       struct mm_struct *mm)
 195 {
 196         struct scif_mmu_notif   *mmn;
 197
 198         mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
 199         scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
 200         schedule_work(&scif_info.misc_work);
 201 }
 202
 203 static int scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 204                                                      struct mm_struct *mm,
 205                                                      unsigned long start,
 206                                                      unsigned long end,
 207                                                      bool blockable)
 208 {
 209         struct scif_mmu_notif   *mmn;
 210
 211         mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
 212         scif_rma_destroy_tcw(mmn, start, end - start);
 213
 214         return 0;
 215 }
 216
 217 static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 218                                                    struct mm_struct *mm,
 219                                                    unsigned long start,
 220                                                    unsigned long end)
 221 {
 222         /*
 223          * Nothing to do here, everything needed was done in
 224          * invalidate_range_start.
 225          */
 226 }
 227
 228 static const struct mmu_notifier_ops scif_mmu_notifier_ops = {
 229         .release = scif_mmu_notifier_release,
 230         .clear_flush_young = NULL,
 231         .invalidate_range_start = scif_mmu_notifier_invalidate_range_start,
 232         .invalidate_range_end = scif_mmu_notifier_invalidate_range_end};
 233
 234 static void scif_ep_unregister_mmu_notifier(struct scif_endpt *ep)
 235 {
 236         struct scif_endpt_rma_info *rma = &ep->rma_info;
 237         struct scif_mmu_notif *mmn = NULL;
 238         struct list_head *item, *tmp;
 239
 240         mutex_lock(&ep->rma_info.mmn_lock);
 241         list_for_each_safe(item, tmp, &rma->mmn_list) {
 242                 mmn = list_entry(item, struct scif_mmu_notif, list);
 243                 mmu_notifier_unregister(&mmn->ep_mmu_notifier, mmn->mm);
 244                 list_del(item);
 245                 kfree(mmn);
 246         }
 247         mutex_unlock(&ep->rma_info.mmn_lock);
 248 }
 249
 250 static void scif_init_mmu_notifier(struct scif_mmu_notif *mmn,
 251                                    struct mm_struct *mm, struct scif_endpt *ep)
 252 {
 253         mmn->ep = ep;
 254         mmn->mm = mm;
 255         mmn->ep_mmu_notifier.ops = &scif_mmu_notifier_ops;
 256         INIT_LIST_HEAD(&mmn->list);
 257         INIT_LIST_HEAD(&mmn->tc_reg_list);
 258 }
 259
 260 static struct scif_mmu_notif *
 261 scif_find_mmu_notifier(struct mm_struct *mm, struct scif_endpt_rma_info *rma)
 262 {
 263         struct scif_mmu_notif *mmn;
 264
 265         list_for_each_entry(mmn, &rma->mmn_list, list)
 266                 if (mmn->mm == mm)
 267                         return mmn;
 268         return NULL;
 269 }
 270
 271 static struct scif_mmu_notif *
 272 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
 273 {
 274         struct scif_mmu_notif *mmn
 275                  = kzalloc(sizeof(*mmn), GFP_KERNEL);
 276
 277         if (!mmn)
 278                 return ERR_PTR(-ENOMEM);
 279
 280         scif_init_mmu_notifier(mmn, current->mm, ep);
 281         if (mmu_notifier_register(&mmn->ep_mmu_notifier, current->mm)) {
 282                 kfree(mmn);
 283                 return ERR_PTR(-EBUSY);
 284         }
 285         list_add(&mmn->list, &ep->rma_info.mmn_list);
 286         return mmn;
 287 }
 288
 289 /*
 290  * Called from the misc thread to destroy temporary cached windows and
 291  * unregister the MMU notifier for the SCIF endpoint.
 292  */
 293 void scif_mmu_notif_handler(struct work_struct *work)
 294 {
 295         struct list_head *pos, *tmpq;
 296         struct scif_endpt *ep;
 297 restart:
 298         scif_rma_destroy_tcw_invalid();
 299         spin_lock(&scif_info.rmalock);
 300         list_for_each_safe(pos, tmpq, &scif_info.mmu_notif_cleanup) {
 301                 ep = list_entry(pos, struct scif_endpt, mmu_list);
 302                 list_del(&ep->mmu_list);
 303                 spin_unlock(&scif_info.rmalock);
 304                 scif_rma_destroy_tcw_ep(ep);
 305                 scif_ep_unregister_mmu_notifier(ep);
 306                 goto restart;
 307         }
 308         spin_unlock(&scif_info.rmalock);
 309 }
 310
 311 static bool scif_is_set_reg_cache(int flags)
 312 {
 313         return !!(flags & SCIF_RMA_USECACHE);
 314 }
 315 #else
 316 static struct scif_mmu_notif *
 317 scif_find_mmu_notifier(struct mm_struct *mm,
 318                        struct scif_endpt_rma_info *rma)
 319 {
 320         return NULL;
 321 }
 322
 323 static struct scif_mmu_notif *
 324 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
 325 {
 326         return NULL;
 327 }
 328
 329 void scif_mmu_notif_handler(struct work_struct *work)
 330 {
 331 }
 332
 333 static bool scif_is_set_reg_cache(int flags)
 334 {
 335         return false;
 336 }
 337
 338 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
 339 {
 340         return false;
 341 }
 342 #endif
 343
 344 /**
 345  * scif_register_temp:
 346  * @epd: End Point Descriptor.
 347  * @addr: virtual address to/from which to copy
 348  * @len: length of range to copy
 349  * @out_offset: computed offset returned by reference.
 350  * @out_window: allocated registered window returned by reference.
 351  *
 352  * Create a temporary registered window. The peer will not know about this
 353  * window. This API is used for scif_vreadfrom()/scif_vwriteto() API's.
 354  */
 355 static int
 356 scif_register_temp(scif_epd_t epd, unsigned long addr, size_t len, int prot,
 357                    off_t *out_offset, struct scif_window **out_window)
 358 {
 359         struct scif_endpt *ep = (struct scif_endpt *)epd;
 360         int err;
 361         scif_pinned_pages_t pinned_pages;
 362         size_t aligned_len;
 363
 364         aligned_len = ALIGN(len, PAGE_SIZE);
 365
 366         err = __scif_pin_pages((void *)(addr & PAGE_MASK),
 367                                aligned_len, &prot, 0, &pinned_pages);
 368         if (err)
 369                 return err;
 370
 371         pinned_pages->prot = prot;
 372
 373         /* Compute the offset for this registration */
 374         err = scif_get_window_offset(ep, 0, 0,
 375                                      aligned_len >> PAGE_SHIFT,
 376                                      (s64 *)out_offset);
 377         if (err)
 378                 goto error_unpin;
 379
 380         /* Allocate and prepare self registration window */
 381         *out_window = scif_create_window(ep, aligned_len >> PAGE_SHIFT,
 382                                         *out_offset, true);
 383         if (!*out_window) {
 384                 scif_free_window_offset(ep, NULL, *out_offset);
 385                 err = -ENOMEM;
 386                 goto error_unpin;
 387         }
 388
 389         (*out_window)->pinned_pages = pinned_pages;
 390         (*out_window)->nr_pages = pinned_pages->nr_pages;
 391         (*out_window)->prot = pinned_pages->prot;
 392
 393         (*out_window)->va_for_temp = addr & PAGE_MASK;
 394         err = scif_map_window(ep->remote_dev, *out_window);
 395         if (err) {
 396                 /* Something went wrong! Rollback */
 397                 scif_destroy_window(ep, *out_window);
 398                 *out_window = NULL;
 399         } else {
 400                 *out_offset |= (addr - (*out_window)->va_for_temp);
 401         }
 402         return err;
 403 error_unpin:
 404         if (err)
 405                 dev_err(&ep->remote_dev->sdev->dev,
 406                         "%s %d err %d\n", __func__, __LINE__, err);
 407         scif_unpin_pages(pinned_pages);
 408         return err;
 409 }
 410
 411 #define SCIF_DMA_TO (3 * HZ)
 412
 413 /*
 414  * scif_sync_dma - Program a DMA without an interrupt descriptor
 415  *
 416  * @dev - The address of the pointer to the device instance used
 417  * for DMA registration.
 418  * @chan - DMA channel to be used.
 419  * @sync_wait: Wait for DMA to complete?
 420  *
 421  * Return 0 on success and -errno on error.
 422  */
 423 static int scif_sync_dma(struct scif_hw_dev *sdev, struct dma_chan *chan,
 424                          bool sync_wait)
 425 {
 426         int err = 0;
 427         struct dma_async_tx_descriptor *tx = NULL;
 428         enum dma_ctrl_flags flags = DMA_PREP_FENCE;
 429         dma_cookie_t cookie;
 430         struct dma_device *ddev;
 431
 432         if (!chan) {
 433                 err = -EIO;
 434                 dev_err(&sdev->dev, "%s %d err %d\n",
 435                         __func__, __LINE__, err);
 436                 return err;
 437         }
 438         ddev = chan->device;
 439
 440         tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
 441         if (!tx) {
 442                 err = -ENOMEM;
 443                 dev_err(&sdev->dev, "%s %d err %d\n",
 444                         __func__, __LINE__, err);
 445                 goto release;
 446         }
 447         cookie = tx->tx_submit(tx);
 448
 449         if (dma_submit_error(cookie)) {
 450                 err = -ENOMEM;
 451                 dev_err(&sdev->dev, "%s %d err %d\n",
 452                         __func__, __LINE__, err);
 453                 goto release;
 454         }
 455         if (!sync_wait) {
 456                 dma_async_issue_pending(chan);
 457         } else {
 458                 if (dma_sync_wait(chan, cookie) == DMA_COMPLETE) {
 459                         err = 0;
 460                 } else {
 461                         err = -EIO;
 462                         dev_err(&sdev->dev, "%s %d err %d\n",
 463                                 __func__, __LINE__, err);
 464                 }
 465         }
 466 release:
 467         return err;
 468 }
 469
 470 static void scif_dma_callback(void *arg)
 471 {
 472         struct completion *done = (struct completion *)arg;
 473
 474         complete(done);
 475 }
 476
 477 #define SCIF_DMA_SYNC_WAIT true
 478 #define SCIF_DMA_POLL BIT(0)
 479 #define SCIF_DMA_INTR BIT(1)
 480
 481 /*
 482  * scif_async_dma - Program a DMA with an interrupt descriptor
 483  *
 484  * @dev - The address of the pointer to the device instance used
 485  * for DMA registration.
 486  * @chan - DMA channel to be used.
 487  * Return 0 on success and -errno on error.
 488  */
 489 static int scif_async_dma(struct scif_hw_dev *sdev, struct dma_chan *chan)
 490 {
 491         int err = 0;
 492         struct dma_device *ddev;
 493         struct dma_async_tx_descriptor *tx = NULL;
 494         enum dma_ctrl_flags flags = DMA_PREP_INTERRUPT | DMA_PREP_FENCE;
 495         DECLARE_COMPLETION_ONSTACK(done_wait);
 496         dma_cookie_t cookie;
 497         enum dma_status status;
 498
 499         if (!chan) {
 500                 err = -EIO;
 501                 dev_err(&sdev->dev, "%s %d err %d\n",
 502                         __func__, __LINE__, err);
 503                 return err;
 504         }
 505         ddev = chan->device;
 506
 507         tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
 508         if (!tx) {
 509                 err = -ENOMEM;
 510                 dev_err(&sdev->dev, "%s %d err %d\n",
 511                         __func__, __LINE__, err);
 512                 goto release;
 513         }
 514         reinit_completion(&done_wait);
 515         tx->callback = scif_dma_callback;
 516         tx->callback_param = &done_wait;
 517         cookie = tx->tx_submit(tx);
 518
 519         if (dma_submit_error(cookie)) {
 520                 err = -ENOMEM;
 521                 dev_err(&sdev->dev, "%s %d err %d\n",
 522                         __func__, __LINE__, err);
 523                 goto release;
 524         }
 525         dma_async_issue_pending(chan);
 526
 527         err = wait_for_completion_timeout(&done_wait, SCIF_DMA_TO);
 528         if (!err) {
 529                 err = -EIO;
 530                 dev_err(&sdev->dev, "%s %d err %d\n",
 531                         __func__, __LINE__, err);
 532                 goto release;
 533         }
 534         err = 0;
 535         status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 536         if (status != DMA_COMPLETE) {
 537                 err = -EIO;
 538                 dev_err(&sdev->dev, "%s %d err %d\n",
 539                         __func__, __LINE__, err);
 540                 goto release;
 541         }
 542 release:
 543         return err;
 544 }
 545
 546 /*
 547  * scif_drain_dma_poll - Drain all outstanding DMA operations for a particular
 548  * DMA channel via polling.
 549  *
 550  * @sdev - The SCIF device
 551  * @chan - DMA channel
 552  * Return 0 on success and -errno on error.
 553  */
 554 static int scif_drain_dma_poll(struct scif_hw_dev *sdev, struct dma_chan *chan)
 555 {
 556         if (!chan)
 557                 return -EINVAL;
 558         return scif_sync_dma(sdev, chan, SCIF_DMA_SYNC_WAIT);
 559 }
 560
 561 /*
 562  * scif_drain_dma_intr - Drain all outstanding DMA operations for a particular
 563  * DMA channel via interrupt based blocking wait.
 564  *
 565  * @sdev - The SCIF device
 566  * @chan - DMA channel
 567  * Return 0 on success and -errno on error.
 568  */
 569 int scif_drain_dma_intr(struct scif_hw_dev *sdev, struct dma_chan *chan)
 570 {
 571         if (!chan)
 572                 return -EINVAL;
 573         return scif_async_dma(sdev, chan);
 574 }
 575
 576 /**
 577  * scif_rma_destroy_windows:
 578  *
 579  * This routine destroys all windows queued for cleanup
 580  */
 581 void scif_rma_destroy_windows(void)
 582 {
 583         struct list_head *item, *tmp;
 584         struct scif_window *window;
 585         struct scif_endpt *ep;
 586         struct dma_chan *chan;
 587
 588         might_sleep();
 589 restart:
 590         spin_lock(&scif_info.rmalock);
 591         list_for_each_safe(item, tmp, &scif_info.rma) {
 592                 window = list_entry(item, struct scif_window,
 593                                     list);
 594                 ep = (struct scif_endpt *)window->ep;
 595                 chan = ep->rma_info.dma_chan;
 596
 597                 list_del_init(&window->list);
 598                 spin_unlock(&scif_info.rmalock);
 599                 if (!chan || !scifdev_alive(ep) ||
 600                     !scif_drain_dma_intr(ep->remote_dev->sdev,
 601                                          ep->rma_info.dma_chan))
 602                         /* Remove window from global list */
 603                         window->unreg_state = OP_COMPLETED;
 604                 else
 605                         dev_warn(&ep->remote_dev->sdev->dev,
 606                                  "DMA engine hung?\n");
 607                 if (window->unreg_state == OP_COMPLETED) {
 608                         if (window->type == SCIF_WINDOW_SELF)
 609                                 scif_destroy_window(ep, window);
 610                         else
 611                                 scif_destroy_remote_window(window);
 612                         atomic_dec(&ep->rma_info.tw_refcount);
 613                 }
 614                 goto restart;
 615         }
 616         spin_unlock(&scif_info.rmalock);
 617 }
 618
 619 /**
 620  * scif_rma_destroy_tcw:
 621  *
 622  * This routine destroys temporary cached registered windows
 623  * which have been queued for cleanup.
 624  */
 625 void scif_rma_destroy_tcw_invalid(void)
 626 {
 627         struct list_head *item, *tmp;
 628         struct scif_window *window;
 629         struct scif_endpt *ep;
 630         struct dma_chan *chan;
 631
 632         might_sleep();
 633 restart:
 634         spin_lock(&scif_info.rmalock);
 635         list_for_each_safe(item, tmp, &scif_info.rma_tc) {
 636                 window = list_entry(item, struct scif_window, list);
 637                 ep = (struct scif_endpt *)window->ep;
 638                 chan = ep->rma_info.dma_chan;
 639                 list_del_init(&window->list);
 640                 spin_unlock(&scif_info.rmalock);
 641                 mutex_lock(&ep->rma_info.rma_lock);
 642                 if (!chan || !scifdev_alive(ep) ||
 643                     !scif_drain_dma_intr(ep->remote_dev->sdev,
 644                                          ep->rma_info.dma_chan)) {
 645                         atomic_sub(window->nr_pages,
 646                                    &ep->rma_info.tcw_total_pages);
 647                         scif_destroy_window(ep, window);
 648                         atomic_dec(&ep->rma_info.tcw_refcount);
 649                 } else {
 650                         dev_warn(&ep->remote_dev->sdev->dev,
 651                                  "DMA engine hung?\n");
 652                 }
 653                 mutex_unlock(&ep->rma_info.rma_lock);
 654                 goto restart;
 655         }
 656         spin_unlock(&scif_info.rmalock);
 657 }
 658
 659 static inline
 660 void *_get_local_va(off_t off, struct scif_window *window, size_t len)
 661 {
 662         int page_nr = (off - window->offset) >> PAGE_SHIFT;
 663         off_t page_off = off & ~PAGE_MASK;
 664         void *va = NULL;
 665
 666         if (window->type == SCIF_WINDOW_SELF) {
 667                 struct page **pages = window->pinned_pages->pages;
 668
 669                 va = page_address(pages[page_nr]) + page_off;
 670         }
 671         return va;
 672 }
 673
 674 static inline
 675 void *ioremap_remote(off_t off, struct scif_window *window,
 676                      size_t len, struct scif_dev *dev,
 677                      struct scif_window_iter *iter)
 678 {
 679         dma_addr_t phys = scif_off_to_dma_addr(window, off, NULL, iter);
 680
 681         /*
 682          * If the DMA address is not card relative then we need the DMA
 683          * addresses to be an offset into the bar. The aperture base was already
 684          * added so subtract it here since scif_ioremap is going to add it again
 685          */
 686         if (!scifdev_self(dev) && window->type == SCIF_WINDOW_PEER &&
 687             dev->sdev->aper && !dev->sdev->card_rel_da)
 688                 phys = phys - dev->sdev->aper->pa;
 689         return scif_ioremap(phys, len, dev);
 690 }
 691
 692 static inline void
 693 iounmap_remote(void *virt, size_t size, struct scif_copy_work *work)
 694 {
 695         scif_iounmap(virt, size, work->remote_dev);
 696 }
 697
 698 /*
 699  * Takes care of ordering issue caused by
 700  * 1. Hardware:  Only in the case of cpu copy from mgmt node to card
 701  * because of WC memory.
 702  * 2. Software: If memcpy reorders copy instructions for optimization.
 703  * This could happen at both mgmt node and card.
 704  */
 705 static inline void
 706 scif_ordered_memcpy_toio(char *dst, const char *src, size_t count)
 707 {
 708         if (!count)
 709                 return;
 710
 711         memcpy_toio((void __iomem __force *)dst, src, --count);
 712         /* Order the last byte with the previous stores */
 713         wmb();
 714         *(dst + count) = *(src + count);
 715 }
 716
 717 static inline void scif_unaligned_cpy_toio(char *dst, const char *src,
 718                                            size_t count, bool ordered)
 719 {
 720         if (ordered)
 721                 scif_ordered_memcpy_toio(dst, src, count);
 722         else
 723                 memcpy_toio((void __iomem __force *)dst, src, count);
 724 }
 725
 726 static inline
 727 void scif_ordered_memcpy_fromio(char *dst, const char *src, size_t count)
 728 {
 729         if (!count)
 730                 return;
 731
 732         memcpy_fromio(dst, (void __iomem __force *)src, --count);
 733         /* Order the last byte with the previous loads */
 734         rmb();
 735         *(dst + count) = *(src + count);
 736 }
 737
 738 static inline void scif_unaligned_cpy_fromio(char *dst, const char *src,
 739                                              size_t count, bool ordered)
 740 {
 741         if (ordered)
 742                 scif_ordered_memcpy_fromio(dst, src, count);
 743         else
 744                 memcpy_fromio(dst, (void __iomem __force *)src, count);
 745 }
 746
 747 #define SCIF_RMA_ERROR_CODE (~(dma_addr_t)0x0)
 748
 749 /*
 750  * scif_off_to_dma_addr:
 751  * Obtain the dma_addr given the window and the offset.
 752  * @window: Registered window.
 753  * @off: Window offset.
 754  * @nr_bytes: Return the number of contiguous bytes till next DMA addr index.
 755  * @index: Return the index of the dma_addr array found.
 756  * @start_off: start offset of index of the dma addr array found.
 757  * The nr_bytes provides the callee an estimate of the maximum possible
 758  * DMA xfer possible while the index/start_off provide faster lookups
 759  * for the next iteration.
 760  */
 761 dma_addr_t scif_off_to_dma_addr(struct scif_window *window, s64 off,
 762                                 size_t *nr_bytes, struct scif_window_iter *iter)
 763 {
 764         int i, page_nr;
 765         s64 start, end;
 766         off_t page_off;
 767
 768         if (window->nr_pages == window->nr_contig_chunks) {
 769                 page_nr = (off - window->offset) >> PAGE_SHIFT;
 770                 page_off = off & ~PAGE_MASK;
 771
 772                 if (nr_bytes)
 773                         *nr_bytes = PAGE_SIZE - page_off;
 774                 return window->dma_addr[page_nr] | page_off;
 775         }
 776         if (iter) {
 777                 i = iter->index;
 778                 start = iter->offset;
 779         } else {
 780                 i =  0;
 781                 start =  window->offset;
 782         }
 783         for (; i < window->nr_contig_chunks; i++) {
 784                 end = start + (window->num_pages[i] << PAGE_SHIFT);
 785                 if (off >= start && off < end) {
 786                         if (iter) {
 787                                 iter->index = i;
 788                                 iter->offset = start;
 789                         }
 790                         if (nr_bytes)
 791                                 *nr_bytes = end - off;
 792                         return (window->dma_addr[i] + (off - start));
 793                 }
 794                 start += (window->num_pages[i] << PAGE_SHIFT);
 795         }
 796         dev_err(scif_info.mdev.this_device,
 797                 "%s %d BUG. Addr not found? window %p off 0x%llx\n",
 798                 __func__, __LINE__, window, off);
 799         return SCIF_RMA_ERROR_CODE;
 800 }
 801
 802 /*
 803  * Copy between rma window and temporary buffer
 804  */
 805 static void scif_rma_local_cpu_copy(s64 offset, struct scif_window *window,
 806                                     u8 *temp, size_t rem_len, bool to_temp)
 807 {
 808         void *window_virt;
 809         size_t loop_len;
 810         int offset_in_page;
 811         s64 end_offset;
 812
 813         offset_in_page = offset & ~PAGE_MASK;
 814         loop_len = PAGE_SIZE - offset_in_page;
 815
 816         if (rem_len < loop_len)
 817                 loop_len = rem_len;
 818
 819         window_virt = _get_local_va(offset, window, loop_len);
 820         if (!window_virt)
 821                 return;
 822         if (to_temp)
 823                 memcpy(temp, window_virt, loop_len);
 824         else
 825                 memcpy(window_virt, temp, loop_len);
 826
 827         offset += loop_len;
 828         temp += loop_len;
 829         rem_len -= loop_len;
 830
 831         end_offset = window->offset +
 832                 (window->nr_pages << PAGE_SHIFT);
 833         while (rem_len) {
 834                 if (offset == end_offset) {
 835                         window = list_next_entry(window, list);
 836                         end_offset = window->offset +
 837                                 (window->nr_pages << PAGE_SHIFT);
 838                 }
 839                 loop_len = min(PAGE_SIZE, rem_len);
 840                 window_virt = _get_local_va(offset, window, loop_len);
 841                 if (!window_virt)
 842                         return;
 843                 if (to_temp)
 844                         memcpy(temp, window_virt, loop_len);
 845                 else
 846                         memcpy(window_virt, temp, loop_len);
 847                 offset  += loop_len;
 848                 temp    += loop_len;
 849                 rem_len -= loop_len;
 850         }
 851 }
 852
 853 /**
 854  * scif_rma_completion_cb:
 855  * @data: RMA cookie
 856  *
 857  * RMA interrupt completion callback.
 858  */
 859 static void scif_rma_completion_cb(void *data)
 860 {
 861         struct scif_dma_comp_cb *comp_cb = data;
 862
 863         /* Free DMA Completion CB. */
 864         if (comp_cb->dst_window)
 865                 scif_rma_local_cpu_copy(comp_cb->dst_offset,
 866                                         comp_cb->dst_window,
 867                                         comp_cb->temp_buf +
 868                                         comp_cb->header_padding,
 869                                         comp_cb->len, false);
 870         scif_unmap_single(comp_cb->temp_phys, comp_cb->sdev,
 871                           SCIF_KMEM_UNALIGNED_BUF_SIZE);
 872         if (comp_cb->is_cache)
 873                 kmem_cache_free(unaligned_cache,
 874                                 comp_cb->temp_buf_to_free);
 875         else
 876                 kfree(comp_cb->temp_buf_to_free);
 877 }
 878
 879 /* Copies between temporary buffer and offsets provided in work */
 880 static int
 881 scif_rma_list_dma_copy_unaligned(struct scif_copy_work *work,
 882                                  u8 *temp, struct dma_chan *chan,
 883                                  bool src_local)
 884 {
 885         struct scif_dma_comp_cb *comp_cb = work->comp_cb;
 886         dma_addr_t window_dma_addr, temp_dma_addr;
 887         dma_addr_t temp_phys = comp_cb->temp_phys;
 888         size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;
 889         int offset_in_ca, ret = 0;
 890         s64 end_offset, offset;
 891         struct scif_window *window;
 892         void *window_virt_addr;
 893         size_t tail_len;
 894         struct dma_async_tx_descriptor *tx;
 895         struct dma_device *dev = chan->device;
 896         dma_cookie_t cookie;
 897
 898         if (src_local) {
 899                 offset = work->dst_offset;
 900                 window = work->dst_window;
 901         } else {
 902                 offset = work->src_offset;
 903                 window = work->src_window;
 904         }
 905
 906         offset_in_ca = offset & (L1_CACHE_BYTES - 1);
 907         if (offset_in_ca) {
 908                 loop_len = L1_CACHE_BYTES - offset_in_ca;
 909                 loop_len = min(loop_len, remaining_len);
 910                 window_virt_addr = ioremap_remote(offset, window,
 911                                                   loop_len,
 912                                                   work->remote_dev,
 913                                                   NULL);
 914                 if (!window_virt_addr)
 915                         return -ENOMEM;
 916                 if (src_local)
 917                         scif_unaligned_cpy_toio(window_virt_addr, temp,
 918                                                 loop_len,
 919                                                 work->ordered &&
 920                                                 !(remaining_len - loop_len));
 921                 else
 922                         scif_unaligned_cpy_fromio(temp, window_virt_addr,
 923                                                   loop_len, work->ordered &&
 924                                                   !(remaining_len - loop_len));
 925                 iounmap_remote(window_virt_addr, loop_len, work);
 926
 927                 offset += loop_len;
 928                 temp += loop_len;
 929                 temp_phys += loop_len;
 930                 remaining_len -= loop_len;
 931         }
 932
 933         offset_in_ca = offset & ~PAGE_MASK;
 934         end_offset = window->offset +
 935                 (window->nr_pages << PAGE_SHIFT);
 936
 937         tail_len = remaining_len & (L1_CACHE_BYTES - 1);
 938         remaining_len -= tail_len;
 939         while (remaining_len) {
 940                 if (offset == end_offset) {
 941                         window = list_next_entry(window, list);
 942                         end_offset = window->offset +
 943                                 (window->nr_pages << PAGE_SHIFT);
 944                 }
 945                 if (scif_is_mgmt_node())
 946                         temp_dma_addr = temp_phys;
 947                 else
 948                         /* Fix if we ever enable IOMMU on the card */
 949                         temp_dma_addr = (dma_addr_t)virt_to_phys(temp);
 950                 window_dma_addr = scif_off_to_dma_addr(window, offset,
 951                                                        &nr_contig_bytes,
 952                                                        NULL);
 953                 loop_len = min(nr_contig_bytes, remaining_len);
 954                 if (src_local) {
 955                         if (work->ordered && !tail_len &&
 956                             !(remaining_len - loop_len) &&
 957                             loop_len != L1_CACHE_BYTES) {
 958                                 /*
 959                                  * Break up the last chunk of the transfer into
 960                                  * two steps. if there is no tail to guarantee
 961                                  * DMA ordering. SCIF_DMA_POLLING inserts
 962                                  * a status update descriptor in step 1 which
 963                                  * acts as a double sided synchronization fence
 964                                  * for the DMA engine to ensure that the last
 965                                  * cache line in step 2 is updated last.
 966                                  */
 967                                 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
 968                                 tx =
 969                                 dev->device_prep_dma_memcpy(chan,
 970                                                             window_dma_addr,
 971                                                             temp_dma_addr,
 972                                                             loop_len -
 973                                                             L1_CACHE_BYTES,
 974                                                             DMA_PREP_FENCE);
 975                                 if (!tx) {
 976                                         ret = -ENOMEM;
 977                                         goto err;
 978                                 }
 979                                 cookie = tx->tx_submit(tx);
 980                                 if (dma_submit_error(cookie)) {
 981                                         ret = -ENOMEM;
 982                                         goto err;
 983                                 }
 984                                 dma_async_issue_pending(chan);
 985                                 offset += (loop_len - L1_CACHE_BYTES);
 986                                 temp_dma_addr += (loop_len - L1_CACHE_BYTES);
 987                                 window_dma_addr += (loop_len - L1_CACHE_BYTES);
 988                                 remaining_len -= (loop_len - L1_CACHE_BYTES);
 989                                 loop_len = remaining_len;
 990
 991                                 /* Step 2) DMA: L1_CACHE_BYTES */
 992                                 tx =
 993                                 dev->device_prep_dma_memcpy(chan,
 994                                                             window_dma_addr,
 995                                                             temp_dma_addr,
 996                                                             loop_len, 0);
 997                                 if (!tx) {
 998                                         ret = -ENOMEM;
 999                                         goto err;
1000                                 }
1001                                 cookie = tx->tx_submit(tx);
1002                                 if (dma_submit_error(cookie)) {
1003                                         ret = -ENOMEM;
1004                                         goto err;
1005                                 }
1006                                 dma_async_issue_pending(chan);
1007                         } else {
1008                                 tx =
1009                                 dev->device_prep_dma_memcpy(chan,
1010                                                             window_dma_addr,
1011                                                             temp_dma_addr,
1012                                                             loop_len, 0);
1013                                 if (!tx) {
1014                                         ret = -ENOMEM;
1015                                         goto err;
1016                                 }
1017                                 cookie = tx->tx_submit(tx);
1018                                 if (dma_submit_error(cookie)) {
1019                                         ret = -ENOMEM;
1020                                         goto err;
1021                                 }
1022                                 dma_async_issue_pending(chan);
1023                         }
1024                 } else {
1025                         tx = dev->device_prep_dma_memcpy(chan, temp_dma_addr,
1026                                         window_dma_addr, loop_len, 0);
1027                         if (!tx) {
1028                                 ret = -ENOMEM;
1029                                 goto err;
1030                         }
1031                         cookie = tx->tx_submit(tx);
1032                         if (dma_submit_error(cookie)) {
1033                                 ret = -ENOMEM;
1034                                 goto err;
1035                         }
1036                         dma_async_issue_pending(chan);
1037                 }
1038                 if (ret < 0)
1039                         goto err;
1040                 offset += loop_len;
1041                 temp += loop_len;
1042                 temp_phys += loop_len;
1043                 remaining_len -= loop_len;
1044                 offset_in_ca = 0;
1045         }
1046         if (tail_len) {
1047                 if (offset == end_offset) {
1048                         window = list_next_entry(window, list);
1049                         end_offset = window->offset +
1050                                 (window->nr_pages << PAGE_SHIFT);
1051                 }
1052                 window_virt_addr = ioremap_remote(offset, window, tail_len,
1053                                                   work->remote_dev,
1054                                                   NULL);
1055                 if (!window_virt_addr)
1056                         return -ENOMEM;
1057                 /*
1058                  * The CPU copy for the tail bytes must be initiated only once
1059                  * previous DMA transfers for this endpoint have completed
1060                  * to guarantee ordering.
1061                  */
1062                 if (work->ordered) {
1063                         struct scif_dev *rdev = work->remote_dev;
1064
1065                         ret = scif_drain_dma_intr(rdev->sdev, chan);
1066                         if (ret)
1067                                 return ret;
1068                 }
1069                 if (src_local)
1070                         scif_unaligned_cpy_toio(window_virt_addr, temp,
1071                                                 tail_len, work->ordered);
1072                 else
1073                         scif_unaligned_cpy_fromio(temp, window_virt_addr,
1074                                                   tail_len, work->ordered);
1075                 iounmap_remote(window_virt_addr, tail_len, work);
1076         }
1077         tx = dev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_INTERRUPT);
1078         if (!tx) {
1079                 ret = -ENOMEM;
1080                 return ret;
1081         }
1082         tx->callback = &scif_rma_completion_cb;
1083         tx->callback_param = comp_cb;
1084         cookie = tx->tx_submit(tx);
1085
1086         if (dma_submit_error(cookie)) {
1087                 ret = -ENOMEM;
1088                 return ret;
1089         }
1090         dma_async_issue_pending(chan);
1091         return 0;
1092 err:
1093         dev_err(scif_info.mdev.this_device,
1094                 "%s %d Desc Prog Failed ret %d\n",
1095                 __func__, __LINE__, ret);
1096         return ret;
1097 }
1098
1099 /*
1100  * _scif_rma_list_dma_copy_aligned:
1101  *
1102  * Traverse all the windows and perform DMA copy.
1103  */
1104 static int _scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1105                                            struct dma_chan *chan)
1106 {
1107         dma_addr_t src_dma_addr, dst_dma_addr;
1108         size_t loop_len, remaining_len, src_contig_bytes = 0;
1109         size_t dst_contig_bytes = 0;
1110         struct scif_window_iter src_win_iter;
1111         struct scif_window_iter dst_win_iter;
1112         s64 end_src_offset, end_dst_offset;
1113         struct scif_window *src_window = work->src_window;
1114         struct scif_window *dst_window = work->dst_window;
1115         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1116         int ret = 0;
1117         struct dma_async_tx_descriptor *tx;
1118         struct dma_device *dev = chan->device;
1119         dma_cookie_t cookie;
1120
1121         remaining_len = work->len;
1122
1123         scif_init_window_iter(src_window, &src_win_iter);
1124         scif_init_window_iter(dst_window, &dst_win_iter);
1125         end_src_offset = src_window->offset +
1126                 (src_window->nr_pages << PAGE_SHIFT);
1127         end_dst_offset = dst_window->offset +
1128                 (dst_window->nr_pages << PAGE_SHIFT);
1129         while (remaining_len) {
1130                 if (src_offset == end_src_offset) {
1131                         src_window = list_next_entry(src_window, list);
1132                         end_src_offset = src_window->offset +
1133                                 (src_window->nr_pages << PAGE_SHIFT);
1134                         scif_init_window_iter(src_window, &src_win_iter);
1135                 }
1136                 if (dst_offset == end_dst_offset) {
1137                         dst_window = list_next_entry(dst_window, list);
1138                         end_dst_offset = dst_window->offset +
1139                                 (dst_window->nr_pages << PAGE_SHIFT);
1140                         scif_init_window_iter(dst_window, &dst_win_iter);
1141                 }
1142
1143                 /* compute dma addresses for transfer */
1144                 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1145                                                     &src_contig_bytes,
1146                                                     &src_win_iter);
1147                 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1148                                                     &dst_contig_bytes,
1149                                                     &dst_win_iter);
1150                 loop_len = min(src_contig_bytes, dst_contig_bytes);
1151                 loop_len = min(loop_len, remaining_len);
1152                 if (work->ordered && !(remaining_len - loop_len)) {
1153                         /*
1154                          * Break up the last chunk of the transfer into two
1155                          * steps to ensure that the last byte in step 2 is
1156                          * updated last.
1157                          */
1158                         /* Step 1) DMA: Body Length - 1 */
1159                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1160                                                          src_dma_addr,
1161                                                          loop_len - 1,
1162                                                          DMA_PREP_FENCE);
1163                         if (!tx) {
1164                                 ret = -ENOMEM;
1165                                 goto err;
1166                         }
1167                         cookie = tx->tx_submit(tx);
1168                         if (dma_submit_error(cookie)) {
1169                                 ret = -ENOMEM;
1170                                 goto err;
1171                         }
1172                         src_offset += (loop_len - 1);
1173                         dst_offset += (loop_len - 1);
1174                         src_dma_addr += (loop_len - 1);
1175                         dst_dma_addr += (loop_len - 1);
1176                         remaining_len -= (loop_len - 1);
1177                         loop_len = remaining_len;
1178
1179                         /* Step 2) DMA: 1 BYTES */
1180                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1181                                         src_dma_addr, loop_len, 0);
1182                         if (!tx) {
1183                                 ret = -ENOMEM;
1184                                 goto err;
1185                         }
1186                         cookie = tx->tx_submit(tx);
1187                         if (dma_submit_error(cookie)) {
1188                                 ret = -ENOMEM;
1189                                 goto err;
1190                         }
1191                         dma_async_issue_pending(chan);
1192                 } else {
1193                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1194                                         src_dma_addr, loop_len, 0);
1195                         if (!tx) {
1196                                 ret = -ENOMEM;
1197                                 goto err;
1198                         }
1199                         cookie = tx->tx_submit(tx);
1200                         if (dma_submit_error(cookie)) {
1201                                 ret = -ENOMEM;
1202                                 goto err;
1203                         }
1204                 }
1205                 src_offset += loop_len;
1206                 dst_offset += loop_len;
1207                 remaining_len -= loop_len;
1208         }
1209         return ret;
1210 err:
1211         dev_err(scif_info.mdev.this_device,
1212                 "%s %d Desc Prog Failed ret %d\n",
1213                 __func__, __LINE__, ret);
1214         return ret;
1215 }
1216
1217 /*
1218  * scif_rma_list_dma_copy_aligned:
1219  *
1220  * Traverse all the windows and perform DMA copy.
1221  */
1222 static int scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1223                                           struct dma_chan *chan)
1224 {
1225         dma_addr_t src_dma_addr, dst_dma_addr;
1226         size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;
1227         size_t dst_contig_bytes = 0;
1228         int src_cache_off;
1229         s64 end_src_offset, end_dst_offset;
1230         struct scif_window_iter src_win_iter;
1231         struct scif_window_iter dst_win_iter;
1232         void *src_virt, *dst_virt;
1233         struct scif_window *src_window = work->src_window;
1234         struct scif_window *dst_window = work->dst_window;
1235         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1236         int ret = 0;
1237         struct dma_async_tx_descriptor *tx;
1238         struct dma_device *dev = chan->device;
1239         dma_cookie_t cookie;
1240
1241         remaining_len = work->len;
1242         scif_init_window_iter(src_window, &src_win_iter);
1243         scif_init_window_iter(dst_window, &dst_win_iter);
1244
1245         src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1246         if (src_cache_off != 0) {
1247                 /* Head */
1248                 loop_len = L1_CACHE_BYTES - src_cache_off;
1249                 loop_len = min(loop_len, remaining_len);
1250                 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1251                 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1252                 if (src_window->type == SCIF_WINDOW_SELF)
1253                         src_virt = _get_local_va(src_offset, src_window,
1254                                                  loop_len);
1255                 else
1256                         src_virt = ioremap_remote(src_offset, src_window,
1257                                                   loop_len,
1258                                                   work->remote_dev, NULL);
1259                 if (!src_virt)
1260                         return -ENOMEM;
1261                 if (dst_window->type == SCIF_WINDOW_SELF)
1262                         dst_virt = _get_local_va(dst_offset, dst_window,
1263                                                  loop_len);
1264                 else
1265                         dst_virt = ioremap_remote(dst_offset, dst_window,
1266                                                   loop_len,
1267                                                   work->remote_dev, NULL);
1268                 if (!dst_virt) {
1269                         if (src_window->type != SCIF_WINDOW_SELF)
1270                                 iounmap_remote(src_virt, loop_len, work);
1271                         return -ENOMEM;
1272                 }
1273                 if (src_window->type == SCIF_WINDOW_SELF)
1274                         scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1275                                                 remaining_len == loop_len ?
1276                                                 work->ordered : false);
1277                 else
1278                         scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,
1279                                                   remaining_len == loop_len ?
1280                                                   work->ordered : false);
1281                 if (src_window->type != SCIF_WINDOW_SELF)
1282                         iounmap_remote(src_virt, loop_len, work);
1283                 if (dst_window->type != SCIF_WINDOW_SELF)
1284                         iounmap_remote(dst_virt, loop_len, work);
1285                 src_offset += loop_len;
1286                 dst_offset += loop_len;
1287                 remaining_len -= loop_len;
1288         }
1289
1290         end_src_offset = src_window->offset +
1291                 (src_window->nr_pages << PAGE_SHIFT);
1292         end_dst_offset = dst_window->offset +
1293                 (dst_window->nr_pages << PAGE_SHIFT);
1294         tail_len = remaining_len & (L1_CACHE_BYTES - 1);
1295         remaining_len -= tail_len;
1296         while (remaining_len) {
1297                 if (src_offset == end_src_offset) {
1298                         src_window = list_next_entry(src_window, list);
1299                         end_src_offset = src_window->offset +
1300                                 (src_window->nr_pages << PAGE_SHIFT);
1301                         scif_init_window_iter(src_window, &src_win_iter);
1302                 }
1303                 if (dst_offset == end_dst_offset) {
1304                         dst_window = list_next_entry(dst_window, list);
1305                         end_dst_offset = dst_window->offset +
1306                                 (dst_window->nr_pages << PAGE_SHIFT);
1307                         scif_init_window_iter(dst_window, &dst_win_iter);
1308                 }
1309
1310                 /* compute dma addresses for transfer */
1311                 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1312                                                     &src_contig_bytes,
1313                                                     &src_win_iter);
1314                 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1315                                                     &dst_contig_bytes,
1316                                                     &dst_win_iter);
1317                 loop_len = min(src_contig_bytes, dst_contig_bytes);
1318                 loop_len = min(loop_len, remaining_len);
1319                 if (work->ordered && !tail_len &&
1320                     !(remaining_len - loop_len)) {
1321                         /*
1322                          * Break up the last chunk of the transfer into two
1323                          * steps. if there is no tail to gurantee DMA ordering.
1324                          * Passing SCIF_DMA_POLLING inserts a status update
1325                          * descriptor in step 1 which acts as a double sided
1326                          * synchronization fence for the DMA engine to ensure
1327                          * that the last cache line in step 2 is updated last.
1328                          */
1329                         /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
1330                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1331                                                          src_dma_addr,
1332                                                          loop_len -
1333                                                          L1_CACHE_BYTES,
1334                                                          DMA_PREP_FENCE);
1335                         if (!tx) {
1336                                 ret = -ENOMEM;
1337                                 goto err;
1338                         }
1339                         cookie = tx->tx_submit(tx);
1340                         if (dma_submit_error(cookie)) {
1341                                 ret = -ENOMEM;
1342                                 goto err;
1343                         }
1344                         dma_async_issue_pending(chan);
1345                         src_offset += (loop_len - L1_CACHE_BYTES);
1346                         dst_offset += (loop_len - L1_CACHE_BYTES);
1347                         src_dma_addr += (loop_len - L1_CACHE_BYTES);
1348                         dst_dma_addr += (loop_len - L1_CACHE_BYTES);
1349                         remaining_len -= (loop_len - L1_CACHE_BYTES);
1350                         loop_len = remaining_len;
1351
1352                         /* Step 2) DMA: L1_CACHE_BYTES */
1353                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1354                                                          src_dma_addr,
1355                                                          loop_len, 0);
1356                         if (!tx) {
1357                                 ret = -ENOMEM;
1358                                 goto err;
1359                         }
1360                         cookie = tx->tx_submit(tx);
1361                         if (dma_submit_error(cookie)) {
1362                                 ret = -ENOMEM;
1363                                 goto err;
1364                         }
1365                         dma_async_issue_pending(chan);
1366                 } else {
1367                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1368                                                          src_dma_addr,
1369                                                          loop_len, 0);
1370                         if (!tx) {
1371                                 ret = -ENOMEM;
1372                                 goto err;
1373                         }
1374                         cookie = tx->tx_submit(tx);
1375                         if (dma_submit_error(cookie)) {
1376                                 ret = -ENOMEM;
1377                                 goto err;
1378                         }
1379                         dma_async_issue_pending(chan);
1380                 }
1381                 src_offset += loop_len;
1382                 dst_offset += loop_len;
1383                 remaining_len -= loop_len;
1384         }
1385         remaining_len = tail_len;
1386         if (remaining_len) {
1387                 loop_len = remaining_len;
1388                 if (src_offset == end_src_offset)
1389                         src_window = list_next_entry(src_window, list);
1390                 if (dst_offset == end_dst_offset)
1391                         dst_window = list_next_entry(dst_window, list);
1392
1393                 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1394                 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1395                 /*
1396                  * The CPU copy for the tail bytes must be initiated only once
1397                  * previous DMA transfers for this endpoint have completed to
1398                  * guarantee ordering.
1399                  */
1400                 if (work->ordered) {
1401                         struct scif_dev *rdev = work->remote_dev;
1402
1403                         ret = scif_drain_dma_poll(rdev->sdev, chan);
1404                         if (ret)
1405                                 return ret;
1406                 }
1407                 if (src_window->type == SCIF_WINDOW_SELF)
1408                         src_virt = _get_local_va(src_offset, src_window,
1409                                                  loop_len);
1410                 else
1411                         src_virt = ioremap_remote(src_offset, src_window,
1412                                                   loop_len,
1413                                                   work->remote_dev, NULL);
1414                 if (!src_virt)
1415                         return -ENOMEM;
1416
1417                 if (dst_window->type == SCIF_WINDOW_SELF)
1418                         dst_virt = _get_local_va(dst_offset, dst_window,
1419                                                  loop_len);
1420                 else
1421                         dst_virt = ioremap_remote(dst_offset, dst_window,
1422                                                   loop_len,
1423                                                   work->remote_dev, NULL);
1424                 if (!dst_virt) {
1425                         if (src_window->type != SCIF_WINDOW_SELF)
1426                                 iounmap_remote(src_virt, loop_len, work);
1427                         return -ENOMEM;
1428                 }
1429
1430                 if (src_window->type == SCIF_WINDOW_SELF)
1431                         scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1432                                                 work->ordered);
1433                 else
1434                         scif_unaligned_cpy_fromio(dst_virt, src_virt,
1435                                                   loop_len, work->ordered);
1436                 if (src_window->type != SCIF_WINDOW_SELF)
1437                         iounmap_remote(src_virt, loop_len, work);
1438
1439                 if (dst_window->type != SCIF_WINDOW_SELF)
1440                         iounmap_remote(dst_virt, loop_len, work);
1441                 remaining_len -= loop_len;
1442         }
1443         return ret;
1444 err:
1445         dev_err(scif_info.mdev.this_device,
1446                 "%s %d Desc Prog Failed ret %d\n",
1447                 __func__, __LINE__, ret);
1448         return ret;
1449 }
1450
1451 /*
1452  * scif_rma_list_cpu_copy:
1453  *
1454  * Traverse all the windows and perform CPU copy.
1455  */
1456 static int scif_rma_list_cpu_copy(struct scif_copy_work *work)
1457 {
1458         void *src_virt, *dst_virt;
1459         size_t loop_len, remaining_len;
1460         int src_page_off, dst_page_off;
1461         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1462         struct scif_window *src_window = work->src_window;
1463         struct scif_window *dst_window = work->dst_window;
1464         s64 end_src_offset, end_dst_offset;
1465         int ret = 0;
1466         struct scif_window_iter src_win_iter;
1467         struct scif_window_iter dst_win_iter;
1468
1469         remaining_len = work->len;
1470
1471         scif_init_window_iter(src_window, &src_win_iter);
1472         scif_init_window_iter(dst_window, &dst_win_iter);
1473         while (remaining_len) {
1474                 src_page_off = src_offset & ~PAGE_MASK;
1475                 dst_page_off = dst_offset & ~PAGE_MASK;
1476                 loop_len = min(PAGE_SIZE -
1477                                max(src_page_off, dst_page_off),
1478                                remaining_len);
1479
1480                 if (src_window->type == SCIF_WINDOW_SELF)
1481                         src_virt = _get_local_va(src_offset, src_window,
1482                                                  loop_len);
1483                 else
1484                         src_virt = ioremap_remote(src_offset, src_window,
1485                                                   loop_len,
1486                                                   work->remote_dev,
1487                                                   &src_win_iter);
1488                 if (!src_virt) {
1489                         ret = -ENOMEM;
1490                         goto error;
1491                 }
1492
1493                 if (dst_window->type == SCIF_WINDOW_SELF)
1494                         dst_virt = _get_local_va(dst_offset, dst_window,
1495                                                  loop_len);
1496                 else
1497                         dst_virt = ioremap_remote(dst_offset, dst_window,
1498                                                   loop_len,
1499                                                   work->remote_dev,
1500                                                   &dst_win_iter);
1501                 if (!dst_virt) {
1502                         if (src_window->type == SCIF_WINDOW_PEER)
1503                                 iounmap_remote(src_virt, loop_len, work);
1504                         ret = -ENOMEM;
1505                         goto error;
1506                 }
1507
1508                 if (work->loopback) {
1509                         memcpy(dst_virt, src_virt, loop_len);
1510                 } else {
1511                         if (src_window->type == SCIF_WINDOW_SELF)
1512                                 memcpy_toio((void __iomem __force *)dst_virt,
1513                                             src_virt, loop_len);
1514                         else
1515                                 memcpy_fromio(dst_virt,
1516                                               (void __iomem __force *)src_virt,
1517                                               loop_len);
1518                 }
1519                 if (src_window->type == SCIF_WINDOW_PEER)
1520                         iounmap_remote(src_virt, loop_len, work);
1521
1522                 if (dst_window->type == SCIF_WINDOW_PEER)
1523                         iounmap_remote(dst_virt, loop_len, work);
1524
1525                 src_offset += loop_len;
1526                 dst_offset += loop_len;
1527                 remaining_len -= loop_len;
1528                 if (remaining_len) {
1529                         end_src_offset = src_window->offset +
1530                                 (src_window->nr_pages << PAGE_SHIFT);
1531                         end_dst_offset = dst_window->offset +
1532                                 (dst_window->nr_pages << PAGE_SHIFT);
1533                         if (src_offset == end_src_offset) {
1534                                 src_window = list_next_entry(src_window, list);
1535                                 scif_init_window_iter(src_window,
1536                                                       &src_win_iter);
1537                         }
1538                         if (dst_offset == end_dst_offset) {
1539                                 dst_window = list_next_entry(dst_window, list);
1540                                 scif_init_window_iter(dst_window,
1541                                                       &dst_win_iter);
1542                         }
1543                 }
1544         }
1545 error:
1546         return ret;
1547 }
1548
1549 static int scif_rma_list_dma_copy_wrapper(struct scif_endpt *epd,
1550                                           struct scif_copy_work *work,
1551                                           struct dma_chan *chan, off_t loffset)
1552 {
1553         int src_cache_off, dst_cache_off;
1554         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1555         u8 *temp = NULL;
1556         bool src_local = true, dst_local = false;
1557         struct scif_dma_comp_cb *comp_cb;
1558         dma_addr_t src_dma_addr, dst_dma_addr;
1559         int err;
1560
1561         if (is_dma_copy_aligned(chan->device, 1, 1, 1))
1562                 return _scif_rma_list_dma_copy_aligned(work, chan);
1563
1564         src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1565         dst_cache_off = dst_offset & (L1_CACHE_BYTES - 1);
1566
1567         if (dst_cache_off == src_cache_off)
1568                 return scif_rma_list_dma_copy_aligned(work, chan);
1569
1570         if (work->loopback)
1571                 return scif_rma_list_cpu_copy(work);
1572         src_dma_addr = __scif_off_to_dma_addr(work->src_window, src_offset);
1573         dst_dma_addr = __scif_off_to_dma_addr(work->dst_window, dst_offset);
1574         src_local = work->src_window->type == SCIF_WINDOW_SELF;
1575         dst_local = work->dst_window->type == SCIF_WINDOW_SELF;
1576
1577         dst_local = dst_local;
1578         /* Allocate dma_completion cb */
1579         comp_cb = kzalloc(sizeof(*comp_cb), GFP_KERNEL);
1580         if (!comp_cb)
1581                 goto error;
1582
1583         work->comp_cb = comp_cb;
1584         comp_cb->cb_cookie = comp_cb;
1585         comp_cb->dma_completion_func = &scif_rma_completion_cb;
1586
1587         if (work->len + (L1_CACHE_BYTES << 1) < SCIF_KMEM_UNALIGNED_BUF_SIZE) {
1588                 comp_cb->is_cache = false;
1589                 /* Allocate padding bytes to align to a cache line */
1590                 temp = kmalloc(work->len + (L1_CACHE_BYTES << 1),
1591                                GFP_KERNEL);
1592                 if (!temp)
1593                         goto free_comp_cb;
1594                 comp_cb->temp_buf_to_free = temp;
1595                 /* kmalloc(..) does not guarantee cache line alignment */
1596                 if (!IS_ALIGNED((u64)temp, L1_CACHE_BYTES))
1597                         temp = PTR_ALIGN(temp, L1_CACHE_BYTES);
1598         } else {
1599                 comp_cb->is_cache = true;
1600                 temp = kmem_cache_alloc(unaligned_cache, GFP_KERNEL);
1601                 if (!temp)
1602                         goto free_comp_cb;
1603                 comp_cb->temp_buf_to_free = temp;
1604         }
1605
1606         if (src_local) {
1607                 temp += dst_cache_off;
1608                 scif_rma_local_cpu_copy(work->src_offset, work->src_window,
1609                                         temp, work->len, true);
1610         } else {
1611                 comp_cb->dst_window = work->dst_window;
1612                 comp_cb->dst_offset = work->dst_offset;
1613                 work->src_offset = work->src_offset - src_cache_off;
1614                 comp_cb->len = work->len;
1615                 work->len = ALIGN(work->len + src_cache_off, L1_CACHE_BYTES);
1616                 comp_cb->header_padding = src_cache_off;
1617         }
1618         comp_cb->temp_buf = temp;
1619
1620         err = scif_map_single(&comp_cb->temp_phys, temp,
1621                               work->remote_dev, SCIF_KMEM_UNALIGNED_BUF_SIZE);
1622         if (err)
1623                 goto free_temp_buf;
1624         comp_cb->sdev = work->remote_dev;
1625         if (scif_rma_list_dma_copy_unaligned(work, temp, chan, src_local) < 0)
1626                 goto free_temp_buf;
1627         if (!src_local)
1628                 work->fence_type = SCIF_DMA_INTR;
1629         return 0;
1630 free_temp_buf:
1631         if (comp_cb->is_cache)
1632                 kmem_cache_free(unaligned_cache, comp_cb->temp_buf_to_free);
1633         else
1634                 kfree(comp_cb->temp_buf_to_free);
1635 free_comp_cb:
1636         kfree(comp_cb);
1637 error:
1638         return -ENOMEM;
1639 }
1640
1641 /**
1642  * scif_rma_copy:
1643  * @epd: end point descriptor.
1644  * @loffset: offset in local registered address space to/from which to copy
1645  * @addr: user virtual address to/from which to copy
1646  * @len: length of range to copy
1647  * @roffset: offset in remote registered address space to/from which to copy
1648  * @flags: flags
1649  * @dir: LOCAL->REMOTE or vice versa.
1650  * @last_chunk: true if this is the last chunk of a larger transfer
1651  *
1652  * Validate parameters, check if src/dst registered ranges requested for copy
1653  * are valid and initiate either CPU or DMA copy.
1654  */
1655 static int scif_rma_copy(scif_epd_t epd, off_t loffset, unsigned long addr,
1656                          size_t len, off_t roffset, int flags,
1657                          enum scif_rma_dir dir, bool last_chunk)
1658 {
1659         struct scif_endpt *ep = (struct scif_endpt *)epd;
1660         struct scif_rma_req remote_req;
1661         struct scif_rma_req req;
1662         struct scif_window *local_window = NULL;
1663         struct scif_window *remote_window = NULL;
1664         struct scif_copy_work copy_work;
1665         bool loopback;
1666         int err = 0;
1667         struct dma_chan *chan;
1668         struct scif_mmu_notif *mmn = NULL;
1669         bool cache = false;
1670         struct device *spdev;
1671
1672         err = scif_verify_epd(ep);
1673         if (err)
1674                 return err;
1675
1676         if (flags && !(flags & (SCIF_RMA_USECPU | SCIF_RMA_USECACHE |
1677                                 SCIF_RMA_SYNC | SCIF_RMA_ORDERED)))
1678                 return -EINVAL;
1679
1680         loopback = scifdev_self(ep->remote_dev) ? true : false;
1681         copy_work.fence_type = ((flags & SCIF_RMA_SYNC) && last_chunk) ?
1682                                 SCIF_DMA_POLL : 0;
1683         copy_work.ordered = !!((flags & SCIF_RMA_ORDERED) && last_chunk);
1684
1685         /* Use CPU for Mgmt node <-> Mgmt node copies */
1686         if (loopback && scif_is_mgmt_node()) {
1687                 flags |= SCIF_RMA_USECPU;
1688                 copy_work.fence_type = 0x0;
1689         }
1690
1691         cache = scif_is_set_reg_cache(flags);
1692
1693         remote_req.out_window = &remote_window;
1694         remote_req.offset = roffset;
1695         remote_req.nr_bytes = len;
1696         /*
1697          * If transfer is from local to remote then the remote window
1698          * must be writeable and vice versa.
1699          */
1700         remote_req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_WRITE : VM_READ;
1701         remote_req.type = SCIF_WINDOW_PARTIAL;
1702         remote_req.head = &ep->rma_info.remote_reg_list;
1703
1704         spdev = scif_get_peer_dev(ep->remote_dev);
1705         if (IS_ERR(spdev)) {
1706                 err = PTR_ERR(spdev);
1707                 return err;
1708         }
1709
1710         if (addr && cache) {
1711                 mutex_lock(&ep->rma_info.mmn_lock);
1712                 mmn = scif_find_mmu_notifier(current->mm, &ep->rma_info);
1713                 if (!mmn)
1714                         mmn = scif_add_mmu_notifier(current->mm, ep);
1715                 mutex_unlock(&ep->rma_info.mmn_lock);
1716                 if (IS_ERR(mmn)) {
1717                         scif_put_peer_dev(spdev);
1718                         return PTR_ERR(mmn);
1719                 }
1720                 cache = cache && !scif_rma_tc_can_cache(ep, len);
1721         }
1722         mutex_lock(&ep->rma_info.rma_lock);
1723         if (addr) {
1724                 req.out_window = &local_window;
1725                 req.nr_bytes = ALIGN(len + (addr & ~PAGE_MASK),
1726                                      PAGE_SIZE);
1727                 req.va_for_temp = addr & PAGE_MASK;
1728                 req.prot = (dir == SCIF_LOCAL_TO_REMOTE ?
1729                             VM_READ : VM_WRITE | VM_READ);
1730                 /* Does a valid local window exist? */
1731                 if (mmn) {
1732                         spin_lock(&ep->rma_info.tc_lock);
1733                         req.head = &mmn->tc_reg_list;
1734                         err = scif_query_tcw(ep, &req);
1735                         spin_unlock(&ep->rma_info.tc_lock);
1736                 }
1737                 if (!mmn || err) {
1738                         err = scif_register_temp(epd, req.va_for_temp,
1739                                                  req.nr_bytes, req.prot,
1740                                                  &loffset, &local_window);
1741                         if (err) {
1742                                 mutex_unlock(&ep->rma_info.rma_lock);
1743                                 goto error;
1744                         }
1745                         if (!cache)
1746                                 goto skip_cache;
1747                         atomic_inc(&ep->rma_info.tcw_refcount);
1748                         atomic_add_return(local_window->nr_pages,
1749                                           &ep->rma_info.tcw_total_pages);
1750                         if (mmn) {
1751                                 spin_lock(&ep->rma_info.tc_lock);
1752                                 scif_insert_tcw(local_window,
1753                                                 &mmn->tc_reg_list);
1754                                 spin_unlock(&ep->rma_info.tc_lock);
1755                         }
1756                 }
1757 skip_cache:
1758                 loffset = local_window->offset +
1759                                 (addr - local_window->va_for_temp);
1760         } else {
1761                 req.out_window = &local_window;
1762                 req.offset = loffset;
1763                 /*
1764                  * If transfer is from local to remote then the self window
1765                  * must be readable and vice versa.
1766                  */
1767                 req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_READ : VM_WRITE;
1768                 req.nr_bytes = len;
1769                 req.type = SCIF_WINDOW_PARTIAL;
1770                 req.head = &ep->rma_info.reg_list;
1771                 /* Does a valid local window exist? */
1772                 err = scif_query_window(&req);
1773                 if (err) {
1774                         mutex_unlock(&ep->rma_info.rma_lock);
1775                         goto error;
1776                 }
1777         }
1778
1779         /* Does a valid remote window exist? */
1780         err = scif_query_window(&remote_req);
1781         if (err) {
1782                 mutex_unlock(&ep->rma_info.rma_lock);
1783                 goto error;
1784         }
1785
1786         /*
1787          * Prepare copy_work for submitting work to the DMA kernel thread
1788          * or CPU copy routine.
1789          */
1790         copy_work.len = len;
1791         copy_work.loopback = loopback;
1792         copy_work.remote_dev = ep->remote_dev;
1793         if (dir == SCIF_LOCAL_TO_REMOTE) {
1794                 copy_work.src_offset = loffset;
1795                 copy_work.src_window = local_window;
1796                 copy_work.dst_offset = roffset;
1797                 copy_work.dst_window = remote_window;
1798         } else {
1799                 copy_work.src_offset = roffset;
1800                 copy_work.src_window = remote_window;
1801                 copy_work.dst_offset = loffset;
1802                 copy_work.dst_window = local_window;
1803         }
1804
1805         if (flags & SCIF_RMA_USECPU) {
1806                 scif_rma_list_cpu_copy(&copy_work);
1807         } else {
1808                 chan = ep->rma_info.dma_chan;
1809                 err = scif_rma_list_dma_copy_wrapper(epd, &copy_work,
1810                                                      chan, loffset);
1811         }
1812         if (addr && !cache)
1813                 atomic_inc(&ep->rma_info.tw_refcount);
1814
1815         mutex_unlock(&ep->rma_info.rma_lock);
1816
1817         if (last_chunk) {
1818                 struct scif_dev *rdev = ep->remote_dev;
1819
1820                 if (copy_work.fence_type == SCIF_DMA_POLL)
1821                         err = scif_drain_dma_poll(rdev->sdev,
1822                                                   ep->rma_info.dma_chan);
1823                 else if (copy_work.fence_type == SCIF_DMA_INTR)
1824                         err = scif_drain_dma_intr(rdev->sdev,
1825                                                   ep->rma_info.dma_chan);
1826         }
1827
1828         if (addr && !cache)
1829                 scif_queue_for_cleanup(local_window, &scif_info.rma);
1830         scif_put_peer_dev(spdev);
1831         return err;
1832 error:
1833         if (err) {
1834                 if (addr && local_window && !cache)
1835                         scif_destroy_window(ep, local_window);
1836                 dev_err(scif_info.mdev.this_device,
1837                         "%s %d err %d len 0x%lx\n",
1838                         __func__, __LINE__, err, len);
1839         }
1840         scif_put_peer_dev(spdev);
1841         return err;
1842 }
1843
1844 int scif_readfrom(scif_epd_t epd, off_t loffset, size_t len,
1845                   off_t roffset, int flags)
1846 {
1847         int err;
1848
1849         dev_dbg(scif_info.mdev.this_device,
1850                 "SCIFAPI readfrom: ep %p loffset 0x%lx len 0x%lx offset 0x%lx flags 0x%x\n",
1851                 epd, loffset, len, roffset, flags);
1852         if (scif_unaligned(loffset, roffset)) {
1853                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1854                         err = scif_rma_copy(epd, loffset, 0x0,
1855                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1856                                             roffset, flags,
1857                                             SCIF_REMOTE_TO_LOCAL, false);
1858                         if (err)
1859                                 goto readfrom_err;
1860                         loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1861                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1862                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1863                 }
1864         }
1865         err = scif_rma_copy(epd, loffset, 0x0, len,
1866                             roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1867 readfrom_err:
1868         return err;
1869 }
1870 EXPORT_SYMBOL_GPL(scif_readfrom);
1871
1872 int scif_writeto(scif_epd_t epd, off_t loffset, size_t len,
1873                  off_t roffset, int flags)
1874 {
1875         int err;
1876
1877         dev_dbg(scif_info.mdev.this_device,
1878                 "SCIFAPI writeto: ep %p loffset 0x%lx len 0x%lx roffset 0x%lx flags 0x%x\n",
1879                 epd, loffset, len, roffset, flags);
1880         if (scif_unaligned(loffset, roffset)) {
1881                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1882                         err = scif_rma_copy(epd, loffset, 0x0,
1883                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1884                                             roffset, flags,
1885                                             SCIF_LOCAL_TO_REMOTE, false);
1886                         if (err)
1887                                 goto writeto_err;
1888                         loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1889                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1890                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1891                 }
1892         }
1893         err = scif_rma_copy(epd, loffset, 0x0, len,
1894                             roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1895 writeto_err:
1896         return err;
1897 }
1898 EXPORT_SYMBOL_GPL(scif_writeto);
1899
1900 int scif_vreadfrom(scif_epd_t epd, void *addr, size_t len,
1901                    off_t roffset, int flags)
1902 {
1903         int err;
1904
1905         dev_dbg(scif_info.mdev.this_device,
1906                 "SCIFAPI vreadfrom: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1907                 epd, addr, len, roffset, flags);
1908         if (scif_unaligned((off_t __force)addr, roffset)) {
1909                 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1910                         flags &= ~SCIF_RMA_USECACHE;
1911
1912                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1913                         err = scif_rma_copy(epd, 0, (u64)addr,
1914                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1915                                             roffset, flags,
1916                                             SCIF_REMOTE_TO_LOCAL, false);
1917                         if (err)
1918                                 goto vreadfrom_err;
1919                         addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1920                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1921                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1922                 }
1923         }
1924         err = scif_rma_copy(epd, 0, (u64)addr, len,
1925                             roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1926 vreadfrom_err:
1927         return err;
1928 }
1929 EXPORT_SYMBOL_GPL(scif_vreadfrom);
1930
1931 int scif_vwriteto(scif_epd_t epd, void *addr, size_t len,
1932                   off_t roffset, int flags)
1933 {
1934         int err;
1935
1936         dev_dbg(scif_info.mdev.this_device,
1937                 "SCIFAPI vwriteto: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1938                 epd, addr, len, roffset, flags);
1939         if (scif_unaligned((off_t __force)addr, roffset)) {
1940                 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1941                         flags &= ~SCIF_RMA_USECACHE;
1942
1943                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1944                         err = scif_rma_copy(epd, 0, (u64)addr,
1945                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1946                                             roffset, flags,
1947                                             SCIF_LOCAL_TO_REMOTE, false);
1948                         if (err)
1949                                 goto vwriteto_err;
1950                         addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1951                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1952                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1953                 }
1954         }
1955         err = scif_rma_copy(epd, 0, (u64)addr, len,
1956                             roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1957 vwriteto_err:
1958         return err;
1959 }
1960 EXPORT_SYMBOL_GPL(scif_vwriteto);