target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch

   1 From 5cb4bc977d933323429050033da9c701b24df43e Mon Sep 17 00:00:00 2001
   2 From: Biwen Li <biwen.li@nxp.com>
   3 Date: Wed, 17 Apr 2019 18:58:23 +0800
   4 Subject: [PATCH] dma: support layerscape
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 This is an integrated patch of dma for layerscape
  10
  11 Signed-off-by: Biwen Li <biwen.li@nxp.com>
  12 Signed-off-by: Catalin Horghidan <catalin.horghidan@nxp.com>
  13 Signed-off-by: Changming Huang <jerry.huang@nxp.com>
  14 Signed-off-by: Horia Geantă <horia.geanta@nxp.com>
  15 Signed-off-by: Ioana Ciornei <ioana.ciornei@nxp.com>
  16 Signed-off-by: jiaheng.fan <jiaheng.fan@nxp.com>
  17 Signed-off-by: Laurentiu Tudor <laurentiu.tudor@nxp.com>
  18 Signed-off-by: Peng Ma <peng.ma@nxp.com>
  19 Signed-off-by: Radu Alexe <radu.alexe@nxp.com>
  20 Signed-off-by: Rajiv Vishwakarma <rajiv.vishwakarma@nxp.com>
  21 Signed-off-by: Tudor Ambarus <tudor-dan.ambarus@nxp.com>
  22 Signed-off-by: Wen He <wen.he_1@nxp.com>
  23 Signed-off-by: Yuan Yao <yao.yuan@nxp.com>
  24 ---
  25  drivers/dma/Kconfig                     |   33 +-
  26  drivers/dma/Makefile                    |    3 +
  27  drivers/dma/caam_dma.c                  |  462 ++++++++
  28  drivers/dma/dpaa2-qdma/Kconfig          |    8 +
  29  drivers/dma/dpaa2-qdma/Makefile         |    8 +
  30  drivers/dma/dpaa2-qdma/dpaa2-qdma.c     |  781 ++++++++++++++
  31  drivers/dma/dpaa2-qdma/dpaa2-qdma.h     |  181 ++++
  32  drivers/dma/dpaa2-qdma/dpdmai.c         |  515 +++++++++
  33  drivers/dma/dpaa2-qdma/fsl_dpdmai.h     |  521 +++++++++
  34  drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h |  222 ++++
  35  drivers/dma/fsl-edma.c                  |   66 +-
  36  drivers/dma/fsl-qdma.c                  | 1278 +++++++++++++++++++++++
  37  12 files changed, 4073 insertions(+), 5 deletions(-)
  38  create mode 100644 drivers/dma/caam_dma.c
  39  create mode 100644 drivers/dma/dpaa2-qdma/Kconfig
  40  create mode 100644 drivers/dma/dpaa2-qdma/Makefile
  41  create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.c
  42  create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.h
  43  create mode 100644 drivers/dma/dpaa2-qdma/dpdmai.c
  44  create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai.h
  45  create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h
  46  create mode 100644 drivers/dma/fsl-qdma.c
  47
  48 --- a/drivers/dma/Kconfig
  49 +++ b/drivers/dma/Kconfig
  50 @@ -129,6 +129,24 @@ config COH901318
  51         help
  52           Enable support for ST-Ericsson COH 901 318 DMA.
  53
  54 +config CRYPTO_DEV_FSL_CAAM_DMA
  55 +       tristate "CAAM DMA engine support"
  56 +       depends on CRYPTO_DEV_FSL_CAAM_JR
  57 +       default n
  58 +       select DMA_ENGINE
  59 +       select ASYNC_CORE
  60 +       select ASYNC_TX_ENABLE_CHANNEL_SWITCH
  61 +       help
  62 +         Selecting this will offload the DMA operations for users of
  63 +         the scatter gather memcopy API to the CAAM via job rings. The
  64 +         CAAM is a hardware module that provides hardware acceleration to
  65 +         cryptographic operations. It has a built-in DMA controller that can
  66 +         be programmed to read/write cryptographic data. This module defines
  67 +         a DMA driver that uses the DMA capabilities of the CAAM.
  68 +
  69 +         To compile this as a module, choose M here: the module
  70 +         will be called caam_dma.
  71 +
  72  config DMA_BCM2835
  73         tristate "BCM2835 DMA engine support"
  74         depends on ARCH_BCM2835
  75 @@ -215,6 +233,20 @@ config FSL_EDMA
  76           multiplexing capability for DMA request sources(slot).
  77           This module can be found on Freescale Vybrid and LS-1 SoCs.
  78
  79 +config FSL_QDMA
  80 +       tristate "NXP Layerscape qDMA engine support"
  81 +       select DMA_ENGINE
  82 +       select DMA_VIRTUAL_CHANNELS
  83 +       select DMA_ENGINE_RAID
  84 +       select ASYNC_TX_ENABLE_CHANNEL_SWITCH
  85 +       help
  86 +         Support the NXP Layerscape qDMA engine with command queue and legacy mode.
  87 +         Channel virtualization is supported through enqueuing of DMA jobs to,
  88 +         or dequeuing DMA jobs from, different work queues.
  89 +         This module can be found on NXP Layerscape SoCs.
  90 +
  91 +source drivers/dma/dpaa2-qdma/Kconfig
  92 +
  93  config FSL_RAID
  94          tristate "Freescale RAID engine Support"
  95          depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH
  96 @@ -600,7 +632,6 @@ config ZX_DMA
  97         help
  98           Support the DMA engine for ZTE ZX family platform devices.
  99
 100 -
 101  # driver files
 102  source "drivers/dma/bestcomm/Kconfig"
 103
 104 --- a/drivers/dma/Makefile
 105 +++ b/drivers/dma/Makefile
 106 @@ -31,7 +31,9 @@ obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o
 107  obj-$(CONFIG_DW_DMAC_CORE) += dw/
 108  obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o
 109  obj-$(CONFIG_FSL_DMA) += fsldma.o
 110 +obj-$(CONFIG_FSL_DPAA2_QDMA) += dpaa2-qdma/
 111  obj-$(CONFIG_FSL_EDMA) += fsl-edma.o
 112 +obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o
 113  obj-$(CONFIG_FSL_RAID) += fsl_raid.o
 114  obj-$(CONFIG_HSU_DMA) += hsu/
 115  obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o
 116 @@ -71,6 +73,7 @@ obj-$(CONFIG_TI_EDMA) += edma.o
 117  obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
 118  obj-$(CONFIG_ZX_DMA) += zx_dma.o
 119  obj-$(CONFIG_ST_FDMA) += st_fdma.o
 120 +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_DMA) += caam_dma.o
 121
 122  obj-y += qcom/
 123  obj-y += xilinx/
 124 --- /dev/null
 125 +++ b/drivers/dma/caam_dma.c
 126 @@ -0,0 +1,462 @@
 127 +/*
 128 + * caam support for SG DMA
 129 + *
 130 + * Copyright 2016 Freescale Semiconductor, Inc
 131 + * Copyright 2017 NXP
 132 + *
 133 + * Redistribution and use in source and binary forms, with or without
 134 + * modification, are permitted provided that the following conditions are met:
 135 + *     * Redistributions of source code must retain the above copyright
 136 + *       notice, this list of conditions and the following disclaimer.
 137 + *     * Redistributions in binary form must reproduce the above copyright
 138 + *       notice, this list of conditions and the following disclaimer in the
 139 + *       documentation and/or other materials provided with the distribution.
 140 + *     * Neither the names of the above-listed copyright holders nor the
 141 + *       names of any contributors may be used to endorse or promote products
 142 + *       derived from this software without specific prior written permission.
 143 + *
 144 + *
 145 + * ALTERNATIVELY, this software may be distributed under the terms of the
 146 + * GNU General Public License ("GPL") as published by the Free Software
 147 + * Foundation, either version 2 of that License or (at your option) any
 148 + * later version.
 149 + *
 150 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 151 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 152 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 153 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
 154 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 155 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 156 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 157 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 158 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 159 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 160 + * POSSIBILITY OF SUCH DAMAGE.
 161 + */
 162 +
 163 +#include <linux/dma-mapping.h>
 164 +#include <linux/dmaengine.h>
 165 +#include <linux/module.h>
 166 +#include <linux/platform_device.h>
 167 +#include <linux/slab.h>
 168 +
 169 +#include "dmaengine.h"
 170 +
 171 +#include "../crypto/caam/regs.h"
 172 +#include "../crypto/caam/jr.h"
 173 +#include "../crypto/caam/error.h"
 174 +#include "../crypto/caam/desc_constr.h"
 175 +
 176 +#define DESC_DMA_MEMCPY_LEN    ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / \
 177 +                                CAAM_CMD_SZ)
 178 +
 179 +/*
 180 + * This is max chunk size of a DMA transfer. If a buffer is larger than this
 181 + * value it is internally broken into chunks of max CAAM_DMA_CHUNK_SIZE bytes
 182 + * and for each chunk a DMA transfer request is issued.
 183 + * This value is the largest number on 16 bits that is a multiple of 256 bytes
 184 + * (the largest configurable CAAM DMA burst size).
 185 + */
 186 +#define CAAM_DMA_CHUNK_SIZE    65280
 187 +
 188 +struct caam_dma_sh_desc {
 189 +       u32 desc[DESC_DMA_MEMCPY_LEN] ____cacheline_aligned;
 190 +       dma_addr_t desc_dma;
 191 +};
 192 +
 193 +/* caam dma extended descriptor */
 194 +struct caam_dma_edesc {
 195 +       struct dma_async_tx_descriptor async_tx;
 196 +       struct list_head node;
 197 +       struct caam_dma_ctx *ctx;
 198 +       dma_addr_t src_dma;
 199 +       dma_addr_t dst_dma;
 200 +       unsigned int src_len;
 201 +       unsigned int dst_len;
 202 +       u32 jd[] ____cacheline_aligned;
 203 +};
 204 +
 205 +/*
 206 + * caam_dma_ctx - per jr/channel context
 207 + * @chan: dma channel used by async_tx API
 208 + * @node: list_head used to attach to the global dma_ctx_list
 209 + * @jrdev: Job Ring device
 210 + * @pending_q: queue of pending (submitted, but not enqueued) jobs
 211 + * @done_not_acked: jobs that have been completed by jr, but maybe not acked
 212 + * @edesc_lock: protects extended descriptor
 213 + */
 214 +struct caam_dma_ctx {
 215 +       struct dma_chan chan;
 216 +       struct list_head node;
 217 +       struct device *jrdev;
 218 +       struct list_head pending_q;
 219 +       struct list_head done_not_acked;
 220 +       spinlock_t edesc_lock;
 221 +};
 222 +
 223 +static struct dma_device *dma_dev;
 224 +static struct caam_dma_sh_desc *dma_sh_desc;
 225 +static LIST_HEAD(dma_ctx_list);
 226 +
 227 +static dma_cookie_t caam_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 228 +{
 229 +       struct caam_dma_edesc *edesc = NULL;
 230 +       struct caam_dma_ctx *ctx = NULL;
 231 +       dma_cookie_t cookie;
 232 +
 233 +       edesc = container_of(tx, struct caam_dma_edesc, async_tx);
 234 +       ctx = container_of(tx->chan, struct caam_dma_ctx, chan);
 235 +
 236 +       spin_lock_bh(&ctx->edesc_lock);
 237 +
 238 +       cookie = dma_cookie_assign(tx);
 239 +       list_add_tail(&edesc->node, &ctx->pending_q);
 240 +
 241 +       spin_unlock_bh(&ctx->edesc_lock);
 242 +
 243 +       return cookie;
 244 +}
 245 +
 246 +static void caam_jr_chan_free_edesc(struct caam_dma_edesc *edesc)
 247 +{
 248 +       struct caam_dma_ctx *ctx = edesc->ctx;
 249 +       struct caam_dma_edesc *_edesc = NULL;
 250 +
 251 +       spin_lock_bh(&ctx->edesc_lock);
 252 +
 253 +       list_add_tail(&edesc->node, &ctx->done_not_acked);
 254 +       list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) {
 255 +               if (async_tx_test_ack(&edesc->async_tx)) {
 256 +                       list_del(&edesc->node);
 257 +                       kfree(edesc);
 258 +               }
 259 +       }
 260 +
 261 +       spin_unlock_bh(&ctx->edesc_lock);
 262 +}
 263 +
 264 +static void caam_dma_done(struct device *dev, u32 *hwdesc, u32 err,
 265 +                         void *context)
 266 +{
 267 +       struct caam_dma_edesc *edesc = context;
 268 +       struct caam_dma_ctx *ctx = edesc->ctx;
 269 +       dma_async_tx_callback callback;
 270 +       void *callback_param;
 271 +
 272 +       if (err)
 273 +               caam_jr_strstatus(ctx->jrdev, err);
 274 +
 275 +       dma_run_dependencies(&edesc->async_tx);
 276 +
 277 +       spin_lock_bh(&ctx->edesc_lock);
 278 +       dma_cookie_complete(&edesc->async_tx);
 279 +       spin_unlock_bh(&ctx->edesc_lock);
 280 +
 281 +       callback = edesc->async_tx.callback;
 282 +       callback_param = edesc->async_tx.callback_param;
 283 +
 284 +       dma_descriptor_unmap(&edesc->async_tx);
 285 +
 286 +       caam_jr_chan_free_edesc(edesc);
 287 +
 288 +       if (callback)
 289 +               callback(callback_param);
 290 +}
 291 +
 292 +static void caam_dma_memcpy_init_job_desc(struct caam_dma_edesc *edesc)
 293 +{
 294 +       u32 *jd = edesc->jd;
 295 +       u32 *sh_desc = dma_sh_desc->desc;
 296 +       dma_addr_t desc_dma = dma_sh_desc->desc_dma;
 297 +
 298 +       /* init the job descriptor */
 299 +       init_job_desc_shared(jd, desc_dma, desc_len(sh_desc), HDR_REVERSE);
 300 +
 301 +       /* set SEQIN PTR */
 302 +       append_seq_in_ptr(jd, edesc->src_dma, edesc->src_len, 0);
 303 +
 304 +       /* set SEQOUT PTR */
 305 +       append_seq_out_ptr(jd, edesc->dst_dma, edesc->dst_len, 0);
 306 +
 307 +       print_hex_dump_debug("caam dma desc@" __stringify(__LINE__) ": ",
 308 +                            DUMP_PREFIX_ADDRESS, 16, 4, jd, desc_bytes(jd), 1);
 309 +}
 310 +
 311 +static struct dma_async_tx_descriptor *
 312 +caam_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
 313 +                    size_t len, unsigned long flags)
 314 +{
 315 +       struct caam_dma_edesc *edesc;
 316 +       struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
 317 +                                               chan);
 318 +
 319 +       edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | GFP_NOWAIT);
 320 +       if (!edesc)
 321 +               return ERR_PTR(-ENOMEM);
 322 +
 323 +       dma_async_tx_descriptor_init(&edesc->async_tx, chan);
 324 +       edesc->async_tx.tx_submit = caam_dma_tx_submit;
 325 +       edesc->async_tx.flags = flags;
 326 +       edesc->async_tx.cookie = -EBUSY;
 327 +
 328 +       edesc->src_dma = src;
 329 +       edesc->src_len = len;
 330 +       edesc->dst_dma = dst;
 331 +       edesc->dst_len = len;
 332 +       edesc->ctx = ctx;
 333 +
 334 +       caam_dma_memcpy_init_job_desc(edesc);
 335 +
 336 +       return &edesc->async_tx;
 337 +}
 338 +
 339 +/* This function can be called in an interrupt context */
 340 +static void caam_dma_issue_pending(struct dma_chan *chan)
 341 +{
 342 +       struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
 343 +                                               chan);
 344 +       struct caam_dma_edesc *edesc, *_edesc;
 345 +
 346 +       spin_lock_bh(&ctx->edesc_lock);
 347 +       list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) {
 348 +               if (caam_jr_enqueue(ctx->jrdev, edesc->jd,
 349 +                                   caam_dma_done, edesc) < 0)
 350 +                       break;
 351 +               list_del(&edesc->node);
 352 +       }
 353 +       spin_unlock_bh(&ctx->edesc_lock);
 354 +}
 355 +
 356 +static void caam_dma_free_chan_resources(struct dma_chan *chan)
 357 +{
 358 +       struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
 359 +                                               chan);
 360 +       struct caam_dma_edesc *edesc, *_edesc;
 361 +
 362 +       spin_lock_bh(&ctx->edesc_lock);
 363 +       list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) {
 364 +               list_del(&edesc->node);
 365 +               kfree(edesc);
 366 +       }
 367 +       list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) {
 368 +               list_del(&edesc->node);
 369 +               kfree(edesc);
 370 +       }
 371 +       spin_unlock_bh(&ctx->edesc_lock);
 372 +}
 373 +
 374 +static int caam_dma_jr_chan_bind(void)
 375 +{
 376 +       struct device *jrdev;
 377 +       struct caam_dma_ctx *ctx;
 378 +       int bonds = 0;
 379 +       int i;
 380 +
 381 +       for (i = 0; i < caam_jr_driver_probed(); i++) {
 382 +               jrdev = caam_jridx_alloc(i);
 383 +               if (IS_ERR(jrdev)) {
 384 +                       pr_err("job ring device %d allocation failed\n", i);
 385 +                       continue;
 386 +               }
 387 +
 388 +               ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 389 +               if (!ctx) {
 390 +                       caam_jr_free(jrdev);
 391 +                       continue;
 392 +               }
 393 +
 394 +               ctx->chan.device = dma_dev;
 395 +               ctx->chan.private = ctx;
 396 +
 397 +               ctx->jrdev = jrdev;
 398 +
 399 +               INIT_LIST_HEAD(&ctx->pending_q);
 400 +               INIT_LIST_HEAD(&ctx->done_not_acked);
 401 +               INIT_LIST_HEAD(&ctx->node);
 402 +               spin_lock_init(&ctx->edesc_lock);
 403 +
 404 +               dma_cookie_init(&ctx->chan);
 405 +
 406 +               /* add the context of this channel to the context list */
 407 +               list_add_tail(&ctx->node, &dma_ctx_list);
 408 +
 409 +               /* add this channel to the device chan list */
 410 +               list_add_tail(&ctx->chan.device_node, &dma_dev->channels);
 411 +
 412 +               bonds++;
 413 +       }
 414 +
 415 +       return bonds;
 416 +}
 417 +
 418 +static inline void caam_jr_dma_free(struct dma_chan *chan)
 419 +{
 420 +       struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx,
 421 +                                               chan);
 422 +
 423 +       list_del(&ctx->node);
 424 +       list_del(&chan->device_node);
 425 +       caam_jr_free(ctx->jrdev);
 426 +       kfree(ctx);
 427 +}
 428 +
 429 +static void set_caam_dma_desc(u32 *desc)
 430 +{
 431 +       u32 *jmp_cmd;
 432 +
 433 +       /* dma shared descriptor */
 434 +       init_sh_desc(desc, HDR_SHARE_NEVER | (1 << HDR_START_IDX_SHIFT));
 435 +
 436 +       /* REG1 = CAAM_DMA_CHUNK_SIZE */
 437 +       append_math_add_imm_u32(desc, REG1, ZERO, IMM, CAAM_DMA_CHUNK_SIZE);
 438 +
 439 +       /* REG0 = SEQINLEN - CAAM_DMA_CHUNK_SIZE */
 440 +       append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, CAAM_DMA_CHUNK_SIZE);
 441 +
 442 +       /*
 443 +        * if (REG0 > 0)
 444 +        *      jmp to LABEL1
 445 +        */
 446 +       jmp_cmd = append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N |
 447 +                             JUMP_COND_MATH_Z);
 448 +
 449 +       /* REG1 = SEQINLEN */
 450 +       append_math_sub(desc, REG1, SEQINLEN, ZERO, CAAM_CMD_SZ);
 451 +
 452 +       /* LABEL1 */
 453 +       set_jump_tgt_here(desc, jmp_cmd);
 454 +
 455 +       /* VARSEQINLEN = REG1 */
 456 +       append_math_add(desc, VARSEQINLEN, REG1, ZERO, CAAM_CMD_SZ);
 457 +
 458 +       /* VARSEQOUTLEN = REG1 */
 459 +       append_math_add(desc, VARSEQOUTLEN, REG1, ZERO, CAAM_CMD_SZ);
 460 +
 461 +       /* do FIFO STORE */
 462 +       append_seq_fifo_store(desc, 0, FIFOST_TYPE_METADATA | LDST_VLF);
 463 +
 464 +       /* do FIFO LOAD */
 465 +       append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
 466 +                            FIFOLD_TYPE_IFIFO | LDST_VLF);
 467 +
 468 +       /*
 469 +        * if (REG0 > 0)
 470 +        *      jmp 0xF8 (after shared desc header)
 471 +        */
 472 +       append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N |
 473 +                   JUMP_COND_MATH_Z | 0xF8);
 474 +
 475 +       print_hex_dump_debug("caam dma shdesc@" __stringify(__LINE__) ": ",
 476 +                            DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 477 +                            1);
 478 +}
 479 +
 480 +static int __init caam_dma_probe(struct platform_device *pdev)
 481 +{
 482 +       struct device *dev = &pdev->dev;
 483 +       struct device *ctrldev = dev->parent;
 484 +       struct dma_chan *chan, *_chan;
 485 +       u32 *sh_desc;
 486 +       int err = -ENOMEM;
 487 +       int bonds;
 488 +
 489 +       if (!caam_jr_driver_probed()) {
 490 +               dev_info(dev, "Defer probing after JR driver probing\n");
 491 +               return -EPROBE_DEFER;
 492 +       }
 493 +
 494 +       dma_dev = kzalloc(sizeof(*dma_dev), GFP_KERNEL);
 495 +       if (!dma_dev)
 496 +               return -ENOMEM;
 497 +
 498 +       dma_sh_desc = kzalloc(sizeof(*dma_sh_desc), GFP_KERNEL | GFP_DMA);
 499 +       if (!dma_sh_desc)
 500 +               goto desc_err;
 501 +
 502 +       sh_desc = dma_sh_desc->desc;
 503 +       set_caam_dma_desc(sh_desc);
 504 +       dma_sh_desc->desc_dma = dma_map_single(ctrldev, sh_desc,
 505 +                                              desc_bytes(sh_desc),
 506 +                                              DMA_TO_DEVICE);
 507 +       if (dma_mapping_error(ctrldev, dma_sh_desc->desc_dma)) {
 508 +               dev_err(dev, "unable to map dma descriptor\n");
 509 +               goto map_err;
 510 +       }
 511 +
 512 +       INIT_LIST_HEAD(&dma_dev->channels);
 513 +
 514 +       bonds = caam_dma_jr_chan_bind();
 515 +       if (!bonds) {
 516 +               err = -ENODEV;
 517 +               goto jr_bind_err;
 518 +       }
 519 +
 520 +       dma_dev->dev = dev;
 521 +       dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 522 +       dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
 523 +       dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
 524 +       dma_dev->device_tx_status = dma_cookie_status;
 525 +       dma_dev->device_issue_pending = caam_dma_issue_pending;
 526 +       dma_dev->device_prep_dma_memcpy = caam_dma_prep_memcpy;
 527 +       dma_dev->device_free_chan_resources = caam_dma_free_chan_resources;
 528 +
 529 +       err = dma_async_device_register(dma_dev);
 530 +       if (err) {
 531 +               dev_err(dev, "Failed to register CAAM DMA engine\n");
 532 +               goto jr_bind_err;
 533 +       }
 534 +
 535 +       dev_info(dev, "caam dma support with %d job rings\n", bonds);
 536 +
 537 +       return err;
 538 +
 539 +jr_bind_err:
 540 +       list_for_each_entry_safe(chan, _chan, &dma_dev->channels, device_node)
 541 +               caam_jr_dma_free(chan);
 542 +
 543 +       dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, desc_bytes(sh_desc),
 544 +                        DMA_TO_DEVICE);
 545 +map_err:
 546 +       kfree(dma_sh_desc);
 547 +desc_err:
 548 +       kfree(dma_dev);
 549 +       return err;
 550 +}
 551 +
 552 +static int caam_dma_remove(struct platform_device *pdev)
 553 +{
 554 +       struct device *dev = &pdev->dev;
 555 +       struct device *ctrldev = dev->parent;
 556 +       struct caam_dma_ctx *ctx, *_ctx;
 557 +
 558 +       dma_async_device_unregister(dma_dev);
 559 +
 560 +       list_for_each_entry_safe(ctx, _ctx, &dma_ctx_list, node) {
 561 +               list_del(&ctx->node);
 562 +               caam_jr_free(ctx->jrdev);
 563 +               kfree(ctx);
 564 +       }
 565 +
 566 +       dma_unmap_single(ctrldev, dma_sh_desc->desc_dma,
 567 +                        desc_bytes(dma_sh_desc->desc), DMA_TO_DEVICE);
 568 +
 569 +       kfree(dma_sh_desc);
 570 +       kfree(dma_dev);
 571 +
 572 +       dev_info(dev, "caam dma support disabled\n");
 573 +       return 0;
 574 +}
 575 +
 576 +static struct platform_driver caam_dma_driver = {
 577 +       .driver = {
 578 +               .name = "caam-dma",
 579 +       },
 580 +       .probe  = caam_dma_probe,
 581 +       .remove = caam_dma_remove,
 582 +};
 583 +module_platform_driver(caam_dma_driver);
 584 +
 585 +MODULE_LICENSE("Dual BSD/GPL");
 586 +MODULE_DESCRIPTION("NXP CAAM support for DMA engine");
 587 +MODULE_AUTHOR("NXP Semiconductors");
 588 +MODULE_ALIAS("platform:caam-dma");
 589 --- /dev/null
 590 +++ b/drivers/dma/dpaa2-qdma/Kconfig
 591 @@ -0,0 +1,8 @@
 592 +menuconfig FSL_DPAA2_QDMA
 593 +       tristate "NXP DPAA2 QDMA"
 594 +       depends on FSL_MC_BUS && FSL_MC_DPIO
 595 +       select DMA_ENGINE
 596 +       select DMA_VIRTUAL_CHANNELS
 597 +       ---help---
 598 +         NXP Data Path Acceleration Architecture 2 QDMA driver,
 599 +         using the NXP MC bus driver.
 600 --- /dev/null
 601 +++ b/drivers/dma/dpaa2-qdma/Makefile
 602 @@ -0,0 +1,8 @@
 603 +#
 604 +# Makefile for the NXP DPAA2 CAAM controllers
 605 +#
 606 +ccflags-y += -DVERSION=\"\"
 607 +
 608 +obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma.o
 609 +
 610 +fsl-dpaa2-qdma-objs    := dpaa2-qdma.o dpdmai.o
 611 --- /dev/null
 612 +++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.c
 613 @@ -0,0 +1,781 @@
 614 +/*
 615 + * drivers/dma/dpaa2-qdma/dpaa2-qdma.c
 616 + *
 617 + * Copyright 2015-2017 NXP Semiconductor, Inc.
 618 + * Author: Changming Huang <jerry.huang@nxp.com>
 619 + *
 620 + * Driver for the NXP QDMA engine with QMan mode.
 621 + * Channel virtualization is supported through enqueuing of DMA jobs to,
 622 + * or dequeuing DMA jobs from different work queues with QMan portal.
 623 + * This module can be found on NXP LS2 SoCs.
 624 + *
 625 + * This program is free software; you can redistribute  it and/or modify it
 626 + * under  the terms of  the GNU General  Public License as published by the
 627 + * Free Software Foundation;  either version 2 of the  License, or (at your
 628 + * option) any later version.
 629 + */
 630 +
 631 +#include <linux/init.h>
 632 +#include <linux/module.h>
 633 +#include <linux/interrupt.h>
 634 +#include <linux/clk.h>
 635 +#include <linux/dma-mapping.h>
 636 +#include <linux/dmapool.h>
 637 +#include <linux/slab.h>
 638 +#include <linux/spinlock.h>
 639 +#include <linux/of.h>
 640 +#include <linux/of_device.h>
 641 +#include <linux/of_address.h>
 642 +#include <linux/of_irq.h>
 643 +#include <linux/of_dma.h>
 644 +#include <linux/types.h>
 645 +#include <linux/delay.h>
 646 +#include <linux/iommu.h>
 647 +#include <linux/sys_soc.h>
 648 +
 649 +#include "../virt-dma.h"
 650 +
 651 +#include <linux/fsl/mc.h>
 652 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h"
 653 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
 654 +#include "fsl_dpdmai_cmd.h"
 655 +#include "fsl_dpdmai.h"
 656 +#include "dpaa2-qdma.h"
 657 +
 658 +static bool smmu_disable = true;
 659 +
 660 +static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan)
 661 +{
 662 +       return container_of(chan, struct dpaa2_qdma_chan, vchan.chan);
 663 +}
 664 +
 665 +static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
 666 +{
 667 +       return container_of(vd, struct dpaa2_qdma_comp, vdesc);
 668 +}
 669 +
 670 +static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan)
 671 +{
 672 +       return 0;
 673 +}
 674 +
 675 +static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan)
 676 +{
 677 +       struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
 678 +       unsigned long flags;
 679 +       LIST_HEAD(head);
 680 +
 681 +       spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags);
 682 +       vchan_get_all_descriptors(&dpaa2_chan->vchan, &head);
 683 +       spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags);
 684 +
 685 +       vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head);
 686 +}
 687 +
 688 +/*
 689 + * Request a command descriptor for enqueue.
 690 + */
 691 +static struct dpaa2_qdma_comp *
 692 +dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan)
 693 +{
 694 +       struct dpaa2_qdma_comp *comp_temp = NULL;
 695 +       unsigned long flags;
 696 +
 697 +       spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
 698 +       if (list_empty(&dpaa2_chan->comp_free)) {
 699 +               spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 700 +               comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
 701 +               if (!comp_temp)
 702 +                       goto err;
 703 +               comp_temp->fd_virt_addr = dma_pool_alloc(dpaa2_chan->fd_pool,
 704 +                               GFP_NOWAIT, &comp_temp->fd_bus_addr);
 705 +               if (!comp_temp->fd_virt_addr)
 706 +                       goto err;
 707 +
 708 +               comp_temp->fl_virt_addr =
 709 +                       (void *)((struct dpaa2_fd *)
 710 +                               comp_temp->fd_virt_addr + 1);
 711 +               comp_temp->fl_bus_addr = comp_temp->fd_bus_addr +
 712 +                                       sizeof(struct dpaa2_fd);
 713 +               comp_temp->desc_virt_addr =
 714 +                       (void *)((struct dpaa2_fl_entry *)
 715 +                               comp_temp->fl_virt_addr + 3);
 716 +               comp_temp->desc_bus_addr = comp_temp->fl_bus_addr +
 717 +                               sizeof(struct dpaa2_fl_entry) * 3;
 718 +
 719 +               comp_temp->qchan = dpaa2_chan;
 720 +               return comp_temp;
 721 +       }
 722 +       comp_temp = list_first_entry(&dpaa2_chan->comp_free,
 723 +                       struct dpaa2_qdma_comp, list);
 724 +       list_del(&comp_temp->list);
 725 +       spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 726 +
 727 +       comp_temp->qchan = dpaa2_chan;
 728 +err:
 729 +       return comp_temp;
 730 +}
 731 +
 732 +static void dpaa2_qdma_populate_fd(uint32_t format,
 733 +               struct dpaa2_qdma_comp *dpaa2_comp)
 734 +{
 735 +       struct dpaa2_fd *fd;
 736 +
 737 +       fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr;
 738 +       memset(fd, 0, sizeof(struct dpaa2_fd));
 739 +
 740 +       /* fd populated */
 741 +       dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr);
 742 +       /* Bypass memory translation, Frame list format, short length disable */
 743 +       /* we need to disable BMT if fsl-mc use iova addr */
 744 +       if (smmu_disable)
 745 +               dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE);
 746 +       dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE);
 747 +
 748 +       dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX);
 749 +}
 750 +
 751 +/* first frame list for descriptor buffer */
 752 +static void dpaa2_qdma_populate_first_framel(
 753 +               struct dpaa2_fl_entry *f_list,
 754 +               struct dpaa2_qdma_comp *dpaa2_comp,
 755 +               bool wrt_changed)
 756 +{
 757 +       struct dpaa2_qdma_sd_d *sdd;
 758 +
 759 +       sdd = (struct dpaa2_qdma_sd_d *)dpaa2_comp->desc_virt_addr;
 760 +       memset(sdd, 0, 2 * (sizeof(*sdd)));
 761 +       /* source and destination descriptor */
 762 +       sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT); /* source descriptor CMD */
 763 +       sdd++;
 764 +
 765 +       /* dest descriptor CMD */
 766 +       if (wrt_changed)
 767 +               sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT);
 768 +       else
 769 +               sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT);
 770 +
 771 +       memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 772 +       /* first frame list to source descriptor */
 773 +
 774 +       dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr);
 775 +       dpaa2_fl_set_len(f_list, 0x20);
 776 +       dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG);
 777 +
 778 +       if (smmu_disable)
 779 +               f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */
 780 +}
 781 +
 782 +/* source and destination frame list */
 783 +static void dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list,
 784 +               dma_addr_t dst, dma_addr_t src, size_t len, uint8_t fmt)
 785 +{
 786 +       /* source frame list to source buffer */
 787 +       memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 788 +
 789 +
 790 +       dpaa2_fl_set_addr(f_list, src);
 791 +       dpaa2_fl_set_len(f_list, len);
 792 +       dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); /* single buffer frame or scatter gather frame */
 793 +       if (smmu_disable)
 794 +               f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */
 795 +
 796 +       f_list++;
 797 +       /* destination frame list to destination buffer */
 798 +       memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 799 +
 800 +       dpaa2_fl_set_addr(f_list, dst);
 801 +       dpaa2_fl_set_len(f_list, len);
 802 +       dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
 803 +       dpaa2_fl_set_final(f_list, QDMA_FL_F); /* single buffer frame or scatter gather frame */
 804 +       if (smmu_disable)
 805 +               f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */
 806 +}
 807 +
 808 +static struct dma_async_tx_descriptor *dpaa2_qdma_prep_memcpy(
 809 +               struct dma_chan *chan, dma_addr_t dst,
 810 +               dma_addr_t src, size_t len, unsigned long flags)
 811 +{
 812 +       struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
 813 +       struct dpaa2_qdma_engine *dpaa2_qdma;
 814 +       struct dpaa2_qdma_comp *dpaa2_comp;
 815 +       struct dpaa2_fl_entry *f_list;
 816 +       bool    wrt_changed;
 817 +       uint32_t format;
 818 +
 819 +       dpaa2_qdma = dpaa2_chan->qdma;
 820 +       dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);
 821 +       wrt_changed = dpaa2_qdma->qdma_wrtype_fixup;
 822 +
 823 +#ifdef LONG_FORMAT
 824 +       format = QDMA_FD_LONG_FORMAT;
 825 +#else
 826 +       format = QDMA_FD_SHORT_FORMAT;
 827 +#endif
 828 +       /* populate Frame descriptor */
 829 +       dpaa2_qdma_populate_fd(format, dpaa2_comp);
 830 +
 831 +       f_list = (struct dpaa2_fl_entry *)dpaa2_comp->fl_virt_addr;
 832 +
 833 +#ifdef LONG_FORMAT
 834 +       /* first frame list for descriptor buffer (logn format) */
 835 +       dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed);
 836 +
 837 +       f_list++;
 838 +#endif
 839 +
 840 +       dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF);
 841 +
 842 +       return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);
 843 +}
 844 +
 845 +static enum dma_status dpaa2_qdma_tx_status(struct dma_chan *chan,
 846 +               dma_cookie_t cookie, struct dma_tx_state *txstate)
 847 +{
 848 +       return dma_cookie_status(chan, cookie, txstate);
 849 +}
 850 +
 851 +static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)
 852 +{
 853 +}
 854 +
 855 +static void dpaa2_qdma_issue_pending(struct dma_chan *chan)
 856 +{
 857 +       struct dpaa2_qdma_comp *dpaa2_comp;
 858 +       struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
 859 +       struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
 860 +       struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv;
 861 +       struct virt_dma_desc *vdesc;
 862 +       struct dpaa2_fd *fd;
 863 +       int err;
 864 +       unsigned long flags;
 865 +
 866 +       spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
 867 +       spin_lock(&dpaa2_chan->vchan.lock);
 868 +       if (vchan_issue_pending(&dpaa2_chan->vchan)) {
 869 +               vdesc = vchan_next_desc(&dpaa2_chan->vchan);
 870 +               if (!vdesc)
 871 +                       goto err_enqueue;
 872 +               dpaa2_comp = to_fsl_qdma_comp(vdesc);
 873 +
 874 +               fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr;
 875 +
 876 +               list_del(&vdesc->node);
 877 +               list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used);
 878 +
 879 +               /* TOBO: priority hard-coded to zero */
 880 +               err = dpaa2_io_service_enqueue_fq(NULL,
 881 +                               priv->tx_queue_attr[0].fqid, fd);
 882 +               if (err) {
 883 +                       list_del(&dpaa2_comp->list);
 884 +                       list_add_tail(&dpaa2_comp->list,
 885 +                               &dpaa2_chan->comp_free);
 886 +               }
 887 +
 888 +       }
 889 +err_enqueue:
 890 +       spin_unlock(&dpaa2_chan->vchan.lock);
 891 +       spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 892 +}
 893 +
 894 +static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev)
 895 +{
 896 +       struct device *dev = &ls_dev->dev;
 897 +       struct dpaa2_qdma_priv *priv;
 898 +       struct dpaa2_qdma_priv_per_prio *ppriv;
 899 +       uint8_t prio_def = DPDMAI_PRIO_NUM;
 900 +       int err;
 901 +       int i;
 902 +
 903 +       priv = dev_get_drvdata(dev);
 904 +
 905 +       priv->dev = dev;
 906 +       priv->dpqdma_id = ls_dev->obj_desc.id;
 907 +
 908 +       /*Get the handle for the DPDMAI this interface is associate with */
 909 +       err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle);
 910 +       if (err) {
 911 +               dev_err(dev, "dpdmai_open() failed\n");
 912 +               return err;
 913 +       }
 914 +       dev_info(dev, "Opened dpdmai object successfully\n");
 915 +
 916 +       err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
 917 +                               &priv->dpdmai_attr);
 918 +       if (err) {
 919 +               dev_err(dev, "dpdmai_get_attributes() failed\n");
 920 +               return err;
 921 +       }
 922 +
 923 +       if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) {
 924 +               dev_err(dev, "DPDMAI major version mismatch\n"
 925 +                            "Found %u.%u, supported version is %u.%u\n",
 926 +                               priv->dpdmai_attr.version.major,
 927 +                               priv->dpdmai_attr.version.minor,
 928 +                               DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
 929 +       }
 930 +
 931 +       if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) {
 932 +               dev_err(dev, "DPDMAI minor version mismatch\n"
 933 +                            "Found %u.%u, supported version is %u.%u\n",
 934 +                               priv->dpdmai_attr.version.major,
 935 +                               priv->dpdmai_attr.version.minor,
 936 +                               DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
 937 +       }
 938 +
 939 +       priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def);
 940 +       ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL);
 941 +       if (!ppriv) {
 942 +               dev_err(dev, "kzalloc for ppriv failed\n");
 943 +               return -1;
 944 +       }
 945 +       priv->ppriv = ppriv;
 946 +
 947 +       for (i = 0; i < priv->num_pairs; i++) {
 948 +               err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 949 +                               i, &priv->rx_queue_attr[i]);
 950 +               if (err) {
 951 +                       dev_err(dev, "dpdmai_get_rx_queue() failed\n");
 952 +                       return err;
 953 +               }
 954 +               ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
 955 +
 956 +               err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 957 +                               i, &priv->tx_queue_attr[i]);
 958 +               if (err) {
 959 +                       dev_err(dev, "dpdmai_get_tx_queue() failed\n");
 960 +                       return err;
 961 +               }
 962 +               ppriv->req_fqid = priv->tx_queue_attr[i].fqid;
 963 +               ppriv->prio = i;
 964 +               ppriv->priv = priv;
 965 +               ppriv++;
 966 +       }
 967 +
 968 +       return 0;
 969 +}
 970 +
 971 +static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx)
 972 +{
 973 +       struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx,
 974 +                       struct dpaa2_qdma_priv_per_prio, nctx);
 975 +       struct dpaa2_qdma_priv *priv = ppriv->priv;
 976 +       struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp;
 977 +       struct dpaa2_qdma_chan *qchan;
 978 +       const struct dpaa2_fd *fd;
 979 +       const struct dpaa2_fd *fd_eq;
 980 +       struct dpaa2_dq *dq;
 981 +       int err;
 982 +       int is_last = 0;
 983 +       uint8_t status;
 984 +       int i;
 985 +       int found;
 986 +       uint32_t n_chans = priv->dpaa2_qdma->n_chans;
 987 +
 988 +       do {
 989 +               err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
 990 +                                               ppriv->store);
 991 +       } while (err);
 992 +
 993 +       while (!is_last) {
 994 +               do {
 995 +                       dq = dpaa2_io_store_next(ppriv->store, &is_last);
 996 +               } while (!is_last && !dq);
 997 +               if (!dq) {
 998 +                       dev_err(priv->dev, "FQID returned no valid frames!\n");
 999 +                       continue;
1000 +               }
1001 +
1002 +               /* obtain FD and process the error */
1003 +               fd = dpaa2_dq_fd(dq);
1004 +
1005 +               status = dpaa2_fd_get_ctrl(fd) & 0xff;
1006 +               if (status)
1007 +                       dev_err(priv->dev, "FD error occurred\n");
1008 +               found = 0;
1009 +               for (i = 0; i < n_chans; i++) {
1010 +                       qchan = &priv->dpaa2_qdma->chans[i];
1011 +                       spin_lock(&qchan->queue_lock);
1012 +                       if (list_empty(&qchan->comp_used)) {
1013 +                               spin_unlock(&qchan->queue_lock);
1014 +                               continue;
1015 +                       }
1016 +                       list_for_each_entry_safe(dpaa2_comp, _comp_tmp,
1017 +                               &qchan->comp_used, list) {
1018 +                               fd_eq = (struct dpaa2_fd *)
1019 +                                       dpaa2_comp->fd_virt_addr;
1020 +
1021 +                               if (le64_to_cpu(fd_eq->simple.addr) ==
1022 +                                               le64_to_cpu(fd->simple.addr)) {
1023 +
1024 +                                       list_del(&dpaa2_comp->list);
1025 +                                       list_add_tail(&dpaa2_comp->list,
1026 +                                                       &qchan->comp_free);
1027 +
1028 +                                       spin_lock(&qchan->vchan.lock);
1029 +                                       vchan_cookie_complete(
1030 +                                               &dpaa2_comp->vdesc);
1031 +                                       spin_unlock(&qchan->vchan.lock);
1032 +                                       found = 1;
1033 +                                       break;
1034 +                               }
1035 +                       }
1036 +                       spin_unlock(&qchan->queue_lock);
1037 +                       if (found)
1038 +                               break;
1039 +               }
1040 +       }
1041 +
1042 +       dpaa2_io_service_rearm(NULL, ctx);
1043 +}
1044 +
1045 +static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv)
1046 +{
1047 +       int err, i, num;
1048 +       struct device *dev = priv->dev;
1049 +       struct dpaa2_qdma_priv_per_prio *ppriv;
1050 +
1051 +       num = priv->num_pairs;
1052 +       ppriv = priv->ppriv;
1053 +       for (i = 0; i < num; i++) {
1054 +               ppriv->nctx.is_cdan = 0;
1055 +               ppriv->nctx.desired_cpu = 1;
1056 +               ppriv->nctx.id = ppriv->rsp_fqid;
1057 +               ppriv->nctx.cb = dpaa2_qdma_fqdan_cb;
1058 +               err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev);
1059 +               if (err) {
1060 +                       dev_err(dev, "Notification register failed\n");
1061 +                       goto err_service;
1062 +               }
1063 +
1064 +               ppriv->store = dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE,
1065 +                                               dev);
1066 +               if (!ppriv->store) {
1067 +                       dev_err(dev, "dpaa2_io_store_create() failed\n");
1068 +                       goto err_store;
1069 +               }
1070 +
1071 +               ppriv++;
1072 +       }
1073 +       return 0;
1074 +
1075 +err_store:
1076 +       dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
1077 +err_service:
1078 +       ppriv--;
1079 +       while (ppriv >= priv->ppriv) {
1080 +               dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
1081 +               dpaa2_io_store_destroy(ppriv->store);
1082 +               ppriv--;
1083 +       }
1084 +       return -1;
1085 +}
1086 +
1087 +static void __cold dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv)
1088 +{
1089 +       struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
1090 +       int i;
1091 +
1092 +       for (i = 0; i < priv->num_pairs; i++) {
1093 +               dpaa2_io_store_destroy(ppriv->store);
1094 +               ppriv++;
1095 +       }
1096 +}
1097 +
1098 +static void __cold dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv)
1099 +{
1100 +       struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
1101 +       struct device *dev = priv->dev;
1102 +       int i;
1103 +
1104 +       for (i = 0; i < priv->num_pairs; i++) {
1105 +               dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
1106 +               ppriv++;
1107 +       }
1108 +}
1109 +
1110 +static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv)
1111 +{
1112 +       int err;
1113 +       struct dpdmai_rx_queue_cfg rx_queue_cfg;
1114 +       struct device *dev = priv->dev;
1115 +       struct dpaa2_qdma_priv_per_prio *ppriv;
1116 +       struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
1117 +       int i, num;
1118 +
1119 +       num = priv->num_pairs;
1120 +       ppriv = priv->ppriv;
1121 +       for (i = 0; i < num; i++) {
1122 +               rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX |
1123 +                                       DPDMAI_QUEUE_OPT_DEST;
1124 +               rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
1125 +               rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO;
1126 +               rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
1127 +               rx_queue_cfg.dest_cfg.priority = ppriv->prio;
1128 +               err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
1129 +                               rx_queue_cfg.dest_cfg.priority, &rx_queue_cfg);
1130 +               if (err) {
1131 +                       dev_err(dev, "dpdmai_set_rx_queue() failed\n");
1132 +                       return err;
1133 +               }
1134 +
1135 +               ppriv++;
1136 +       }
1137 +
1138 +       return 0;
1139 +}
1140 +
1141 +static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv)
1142 +{
1143 +       int err = 0;
1144 +       struct device *dev = priv->dev;
1145 +       struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
1146 +       struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
1147 +       int i;
1148 +
1149 +       for (i = 0; i < priv->num_pairs; i++) {
1150 +               ppriv->nctx.qman64 = 0;
1151 +               ppriv->nctx.dpio_id = 0;
1152 +               ppriv++;
1153 +       }
1154 +
1155 +       err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle);
1156 +       if (err)
1157 +               dev_err(dev, "dpdmai_reset() failed\n");
1158 +
1159 +       return err;
1160 +}
1161 +
1162 +static void __cold dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan,
1163 +                                               struct list_head *head)
1164 +{
1165 +       struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp;
1166 +       /* free the QDMA comp resource */
1167 +       list_for_each_entry_safe(comp_tmp, _comp_tmp,
1168 +                       head, list) {
1169 +               dma_pool_free(qchan->fd_pool,
1170 +                       comp_tmp->fd_virt_addr,
1171 +                       comp_tmp->fd_bus_addr);
1172 +               list_del(&comp_tmp->list);
1173 +               kfree(comp_tmp);
1174 +       }
1175 +
1176 +}
1177 +
1178 +static void __cold dpaa2_dpdmai_free_channels(
1179 +               struct dpaa2_qdma_engine *dpaa2_qdma)
1180 +{
1181 +       struct dpaa2_qdma_chan *qchan;
1182 +       int num, i;
1183 +
1184 +       num = dpaa2_qdma->n_chans;
1185 +       for (i = 0; i < num; i++) {
1186 +               qchan = &dpaa2_qdma->chans[i];
1187 +               dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used);
1188 +               dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free);
1189 +               dma_pool_destroy(qchan->fd_pool);
1190 +       }
1191 +}
1192 +
1193 +static int dpaa2_dpdmai_alloc_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
1194 +{
1195 +       struct dpaa2_qdma_chan *dpaa2_chan;
1196 +       struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev;
1197 +       int i;
1198 +
1199 +       INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels);
1200 +       for (i = 0; i < dpaa2_qdma->n_chans; i++) {
1201 +               dpaa2_chan = &dpaa2_qdma->chans[i];
1202 +               dpaa2_chan->qdma = dpaa2_qdma;
1203 +               dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc;
1204 +               vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev);
1205 +
1206 +               dpaa2_chan->fd_pool = dma_pool_create("fd_pool",
1207 +                                       dev, FD_POOL_SIZE, 32, 0);
1208 +               if (!dpaa2_chan->fd_pool)
1209 +                       return -1;
1210 +
1211 +               spin_lock_init(&dpaa2_chan->queue_lock);
1212 +               INIT_LIST_HEAD(&dpaa2_chan->comp_used);
1213 +               INIT_LIST_HEAD(&dpaa2_chan->comp_free);
1214 +       }
1215 +       return 0;
1216 +}
1217 +
1218 +static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)
1219 +{
1220 +       struct dpaa2_qdma_priv *priv;
1221 +       struct device *dev = &dpdmai_dev->dev;
1222 +       struct dpaa2_qdma_engine *dpaa2_qdma;
1223 +       int err;
1224 +
1225 +       priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1226 +       if (!priv)
1227 +               return -ENOMEM;
1228 +       dev_set_drvdata(dev, priv);
1229 +       priv->dpdmai_dev = dpdmai_dev;
1230 +
1231 +       priv->iommu_domain = iommu_get_domain_for_dev(dev);
1232 +       if (priv->iommu_domain)
1233 +               smmu_disable = false;
1234 +
1235 +       /* obtain a MC portal */
1236 +       err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io);
1237 +       if (err) {
1238 +               if (err == -ENXIO)
1239 +                       err = -EPROBE_DEFER;
1240 +               else
1241 +                       dev_err(dev, "MC portal allocation failed\n");
1242 +               goto err_mcportal;
1243 +       }
1244 +
1245 +       /* DPDMAI initialization */
1246 +       err = dpaa2_qdma_setup(dpdmai_dev);
1247 +       if (err) {
1248 +               dev_err(dev, "dpaa2_dpdmai_setup() failed\n");
1249 +               goto err_dpdmai_setup;
1250 +       }
1251 +
1252 +       /* DPIO */
1253 +       err = dpaa2_qdma_dpio_setup(priv);
1254 +       if (err) {
1255 +               dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n");
1256 +               goto err_dpio_setup;
1257 +       }
1258 +
1259 +       /* DPDMAI binding to DPIO */
1260 +       err = dpaa2_dpdmai_bind(priv);
1261 +       if (err) {
1262 +               dev_err(dev, "dpaa2_dpdmai_bind() failed\n");
1263 +               goto err_bind;
1264 +       }
1265 +
1266 +       /* DPDMAI enable */
1267 +       err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);
1268 +       if (err) {
1269 +               dev_err(dev, "dpdmai_enable() faile\n");
1270 +               goto err_enable;
1271 +       }
1272 +
1273 +       dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL);
1274 +       if (!dpaa2_qdma) {
1275 +               err = -ENOMEM;
1276 +               goto err_eng;
1277 +       }
1278 +
1279 +       priv->dpaa2_qdma = dpaa2_qdma;
1280 +       dpaa2_qdma->priv = priv;
1281 +
1282 +       dpaa2_qdma->n_chans = NUM_CH;
1283 +
1284 +       err = dpaa2_dpdmai_alloc_channels(dpaa2_qdma);
1285 +       if (err) {
1286 +               dev_err(dev, "QDMA alloc channels faile\n");
1287 +               goto err_reg;
1288 +       }
1289 +
1290 +       if (soc_device_match(soc_fixup_tuning))
1291 +               dpaa2_qdma->qdma_wrtype_fixup = true;
1292 +       else
1293 +               dpaa2_qdma->qdma_wrtype_fixup = false;
1294 +
1295 +       dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask);
1296 +       dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask);
1297 +       dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask);
1298 +
1299 +       dpaa2_qdma->dma_dev.dev = dev;
1300 +       dpaa2_qdma->dma_dev.device_alloc_chan_resources
1301 +               = dpaa2_qdma_alloc_chan_resources;
1302 +       dpaa2_qdma->dma_dev.device_free_chan_resources
1303 +               = dpaa2_qdma_free_chan_resources;
1304 +       dpaa2_qdma->dma_dev.device_tx_status = dpaa2_qdma_tx_status;
1305 +       dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy;
1306 +       dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending;
1307 +
1308 +       err = dma_async_device_register(&dpaa2_qdma->dma_dev);
1309 +       if (err) {
1310 +               dev_err(dev, "Can't register NXP QDMA engine.\n");
1311 +               goto err_reg;
1312 +       }
1313 +
1314 +       return 0;
1315 +
1316 +err_reg:
1317 +       dpaa2_dpdmai_free_channels(dpaa2_qdma);
1318 +       kfree(dpaa2_qdma);
1319 +err_eng:
1320 +       dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle);
1321 +err_enable:
1322 +       dpaa2_dpdmai_dpio_unbind(priv);
1323 +err_bind:
1324 +       dpaa2_dpmai_store_free(priv);
1325 +       dpaa2_dpdmai_dpio_free(priv);
1326 +err_dpio_setup:
1327 +       dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle);
1328 +err_dpdmai_setup:
1329 +       fsl_mc_portal_free(priv->mc_io);
1330 +err_mcportal:
1331 +       kfree(priv->ppriv);
1332 +       kfree(priv);
1333 +       dev_set_drvdata(dev, NULL);
1334 +       return err;
1335 +}
1336 +
1337 +static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev)
1338 +{
1339 +       struct device *dev;
1340 +       struct dpaa2_qdma_priv *priv;
1341 +       struct dpaa2_qdma_engine *dpaa2_qdma;
1342 +
1343 +       dev = &ls_dev->dev;
1344 +       priv = dev_get_drvdata(dev);
1345 +       dpaa2_qdma = priv->dpaa2_qdma;
1346 +
1347 +       dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
1348 +       dpaa2_dpdmai_dpio_unbind(priv);
1349 +       dpaa2_dpmai_store_free(priv);
1350 +       dpaa2_dpdmai_dpio_free(priv);
1351 +       dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
1352 +       fsl_mc_portal_free(priv->mc_io);
1353 +       dev_set_drvdata(dev, NULL);
1354 +       dpaa2_dpdmai_free_channels(dpaa2_qdma);
1355 +
1356 +       dma_async_device_unregister(&dpaa2_qdma->dma_dev);
1357 +       kfree(priv);
1358 +       kfree(dpaa2_qdma);
1359 +
1360 +       return 0;
1361 +}
1362 +
1363 +static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = {
1364 +       {
1365 +               .vendor = FSL_MC_VENDOR_FREESCALE,
1366 +               .obj_type = "dpdmai",
1367 +       },
1368 +       { .vendor = 0x0 }
1369 +};
1370 +
1371 +static struct fsl_mc_driver dpaa2_qdma_driver = {
1372 +       .driver         = {
1373 +               .name   = "dpaa2-qdma",
1374 +               .owner  = THIS_MODULE,
1375 +       },
1376 +       .probe          = dpaa2_qdma_probe,
1377 +       .remove         = dpaa2_qdma_remove,
1378 +       .match_id_table = dpaa2_qdma_id_table
1379 +};
1380 +
1381 +static int __init dpaa2_qdma_driver_init(void)
1382 +{
1383 +       return fsl_mc_driver_register(&(dpaa2_qdma_driver));
1384 +}
1385 +late_initcall(dpaa2_qdma_driver_init);
1386 +
1387 +static void __exit fsl_qdma_exit(void)
1388 +{
1389 +       fsl_mc_driver_unregister(&(dpaa2_qdma_driver));
1390 +}
1391 +module_exit(fsl_qdma_exit);
1392 +
1393 +MODULE_DESCRIPTION("NXP DPAA2 qDMA driver");
1394 +MODULE_LICENSE("Dual BSD/GPL");
1395 --- /dev/null
1396 +++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.h
1397 @@ -0,0 +1,181 @@
1398 +/* Copyright 2015 NXP Semiconductor Inc.
1399 + *
1400 + * Redistribution and use in source and binary forms, with or without
1401 + * modification, are permitted provided that the following conditions are met:
1402 + *     * Redistributions of source code must retain the above copyright
1403 + *     notice, this list of conditions and the following disclaimer.
1404 + *     * Redistributions in binary form must reproduce the above copyright
1405 + *      notice, this list of conditions and the following disclaimer in the
1406 + *      documentation and/or other materials provided with the distribution.
1407 + *     * Neither the name of NXP Semiconductor nor the
1408 + *      names of its contributors may be used to endorse or promote products
1409 + *      derived from this software without specific prior written permission.
1410 + *
1411 + *
1412 + * ALTERNATIVELY, this software may be distributed under the terms of the
1413 + * GNU General Public License ("GPL") as published by the Free Software
1414 + * Foundation, either version 2 of that License or (at your option) any
1415 + * later version.
1416 + *
1417 + * THIS SOFTWARE IS PROVIDED BY NXP Semiconductor ``AS IS'' AND ANY
1418 + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1419 + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1420 + * DISCLAIMED. IN NO EVENT SHALL NXP Semiconductor BE LIABLE FOR ANY
1421 + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
1422 + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
1423 + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
1424 + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1425 + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1426 + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1427 + */
1428 +
1429 +#ifndef __DPAA2_QDMA_H
1430 +#define __DPAA2_QDMA_H
1431 +
1432 +#define LONG_FORMAT 1
1433 +
1434 +#define DPAA2_QDMA_STORE_SIZE 16
1435 +#define NUM_CH 8
1436 +
1437 +#define QDMA_DMR_OFFSET        0x0
1438 +#define QDMA_DQ_EN (0 << 30)
1439 +#define QDMA_DQ_DIS (1 << 30)
1440 +
1441 +#define QDMA_DSR_M_OFFSET 0x10004
1442 +
1443 +struct dpaa2_qdma_sd_d {
1444 +       uint32_t rsv:32;
1445 +       union {
1446 +               struct {
1447 +                       uint32_t ssd:12; /* souce stride distance */
1448 +                       uint32_t sss:12; /* souce stride size */
1449 +                       uint32_t rsv1:8;
1450 +               } sdf;
1451 +               struct {
1452 +                       uint32_t dsd:12; /* Destination stride distance */
1453 +                       uint32_t dss:12; /* Destination stride size */
1454 +                       uint32_t rsv2:8;
1455 +               } ddf;
1456 +       } df;
1457 +       uint32_t rbpcmd;        /* Route-by-port command */
1458 +       uint32_t cmd;
1459 +} __attribute__((__packed__));
1460 +/* Source descriptor command read transaction type for RBP=0:
1461 + coherent copy of cacheable memory */
1462 +#define QDMA_SD_CMD_RDTTYPE_COHERENT (0xb << 28)
1463 +/* Destination descriptor command write transaction type for RBP=0:
1464 + coherent copy of cacheable memory */
1465 +#define QDMA_DD_CMD_WRTTYPE_COHERENT (0x6 << 28)
1466 +#define LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT (0xb << 28)
1467 +
1468 +#define QMAN_FD_FMT_ENABLE (1) /* frame list table enable */
1469 +#define QMAN_FD_BMT_ENABLE (1 << 15) /* bypass memory translation */
1470 +#define QMAN_FD_BMT_DISABLE (0 << 15) /* bypass memory translation */
1471 +#define QMAN_FD_SL_DISABLE (0 << 14) /* short lengthe disabled */
1472 +#define QMAN_FD_SL_ENABLE (1 << 14) /* short lengthe enabled */
1473 +
1474 +#define QDMA_FINAL_BIT_DISABLE (0 << 31) /* final bit disable */
1475 +#define QDMA_FINAL_BIT_ENABLE (1 << 31) /* final bit enable */
1476 +
1477 +#define QDMA_FD_SHORT_FORMAT (1 << 11) /* short format */
1478 +#define QDMA_FD_LONG_FORMAT (0 << 11) /* long format */
1479 +#define QDMA_SER_DISABLE (0 << 8) /* no notification */
1480 +#define QDMA_SER_CTX (1 << 8) /* notification by FQD_CTX[fqid] */
1481 +#define QDMA_SER_DEST (2 << 8) /* notification by destination desc */
1482 +#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */
1483 +#define QDMA_FD_SPF_ENALBE (1 << 30) /* source prefetch enable */
1484 +
1485 +#define QMAN_FD_VA_ENABLE (1 << 14)  /* Address used is virtual address */
1486 +#define QMAN_FD_VA_DISABLE (0 << 14)/* Address used is a real address */
1487 +#define QMAN_FD_CBMT_ENABLE (1 << 15) /* Flow Context: 49bit physical address */
1488 +#define QMAN_FD_CBMT_DISABLE (0 << 15) /* Flow Context: 64bit virtual address */
1489 +#define QMAN_FD_SC_DISABLE (0 << 27) /* stashing control */
1490 +
1491 +#define QDMA_FL_FMT_SBF (0x0) /* Single buffer frame */
1492 +#define QDMA_FL_FMT_SGE 0x2 /* Scatter gather frame */
1493 +#define QDMA_FL_BMT_ENABLE (0x1 << 15)/* enable bypass memory translation */
1494 +#define QDMA_FL_BMT_DISABLE 0x0 /* enable bypass memory translation */
1495 +#define QDMA_FL_SL_LONG (0x0 << 2)/* long length */
1496 +#define QDMA_FL_SL_SHORT 0x1 /* short length */
1497 +#define QDMA_FL_F (0x1)/* last frame list bit */
1498 +/*Description of Frame list table structure*/
1499 +
1500 +struct dpaa2_qdma_chan {
1501 +       struct virt_dma_chan            vchan;
1502 +       struct virt_dma_desc            vdesc;
1503 +       enum dma_status                 status;
1504 +       struct dpaa2_qdma_engine        *qdma;
1505 +
1506 +       struct mutex            dpaa2_queue_mutex;
1507 +       spinlock_t              queue_lock;
1508 +       struct dma_pool         *fd_pool;
1509 +
1510 +       struct list_head        comp_used;
1511 +       struct list_head        comp_free;
1512 +
1513 +};
1514 +
1515 +struct dpaa2_qdma_comp {
1516 +       dma_addr_t              fd_bus_addr;
1517 +       dma_addr_t              fl_bus_addr;
1518 +       dma_addr_t              desc_bus_addr;
1519 +       void                    *fd_virt_addr;
1520 +       void                    *fl_virt_addr;
1521 +       void                    *desc_virt_addr;
1522 +       struct dpaa2_qdma_chan  *qchan;
1523 +       struct virt_dma_desc    vdesc;
1524 +       struct list_head        list;
1525 +};
1526 +
1527 +struct dpaa2_qdma_engine {
1528 +       struct dma_device       dma_dev;
1529 +       u32                     n_chans;
1530 +       struct dpaa2_qdma_chan  chans[NUM_CH];
1531 +       bool                    qdma_wrtype_fixup;
1532 +
1533 +       struct dpaa2_qdma_priv *priv;
1534 +};
1535 +
1536 +/*
1537 + * dpaa2_qdma_priv - driver private data
1538 + */
1539 +struct dpaa2_qdma_priv {
1540 +       int dpqdma_id;
1541 +
1542 +       struct iommu_domain *iommu_domain;
1543 +       struct dpdmai_attr dpdmai_attr;
1544 +       struct device *dev;
1545 +       struct fsl_mc_io *mc_io;
1546 +       struct fsl_mc_device *dpdmai_dev;
1547 +
1548 +       struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM];
1549 +       struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_PRIO_NUM];
1550 +
1551 +       uint8_t num_pairs;
1552 +
1553 +       struct dpaa2_qdma_engine *dpaa2_qdma;
1554 +       struct dpaa2_qdma_priv_per_prio *ppriv;
1555 +};
1556 +
1557 +struct dpaa2_qdma_priv_per_prio {
1558 +       int req_fqid;
1559 +       int rsp_fqid;
1560 +       int prio;
1561 +
1562 +       struct dpaa2_io_store *store;
1563 +       struct dpaa2_io_notification_ctx nctx;
1564 +
1565 +       struct dpaa2_qdma_priv *priv;
1566 +};
1567 +
1568 +static struct soc_device_attribute soc_fixup_tuning[] = {
1569 +       { .family = "QorIQ LX2160A"},
1570 +       { },
1571 +};
1572 +
1573 +/* FD pool size: one FD + 3 Frame list + 2 source/destination descriptor */
1574 +#define FD_POOL_SIZE (sizeof(struct dpaa2_fd) + \
1575 +               sizeof(struct dpaa2_fl_entry) * 3 + \
1576 +               sizeof(struct dpaa2_qdma_sd_d) * 2)
1577 +
1578 +#endif /* __DPAA2_QDMA_H */
1579 --- /dev/null
1580 +++ b/drivers/dma/dpaa2-qdma/dpdmai.c
1581 @@ -0,0 +1,515 @@
1582 +/* Copyright 2013-2015 Freescale Semiconductor Inc.
1583 + *
1584 + * Redistribution and use in source and binary forms, with or without
1585 + * modification, are permitted provided that the following conditions are met:
1586 + * * Redistributions of source code must retain the above copyright
1587 + * notice, this list of conditions and the following disclaimer.
1588 + * * Redistributions in binary form must reproduce the above copyright
1589 + * notice, this list of conditions and the following disclaimer in the
1590 + * documentation and/or other materials provided with the distribution.
1591 + * * Neither the name of the above-listed copyright holders nor the
1592 + * names of any contributors may be used to endorse or promote products
1593 + * derived from this software without specific prior written permission.
1594 + *
1595 + *
1596 + * ALTERNATIVELY, this software may be distributed under the terms of the
1597 + * GNU General Public License ("GPL") as published by the Free Software
1598 + * Foundation, either version 2 of that License or (at your option) any
1599 + * later version.
1600 + *
1601 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1602 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1603 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1604 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
1605 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
1606 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
1607 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
1608 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
1609 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
1610 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1611 + * POSSIBILITY OF SUCH DAMAGE.
1612 + */
1613 +#include <linux/types.h>
1614 +#include <linux/io.h>
1615 +#include "fsl_dpdmai.h"
1616 +#include "fsl_dpdmai_cmd.h"
1617 +#include <linux/fsl/mc.h>
1618 +
1619 +struct dpdmai_cmd_open {
1620 +       __le32 dpdmai_id;
1621 +};
1622 +
1623 +struct dpdmai_rsp_get_attributes {
1624 +       __le32 id;
1625 +       u8 num_of_priorities;
1626 +       u8 pad0[3];
1627 +       __le16 major;
1628 +       __le16 minor;
1629 +};
1630 +
1631 +
1632 +struct dpdmai_cmd_queue {
1633 +       __le32 dest_id;
1634 +       u8 priority;
1635 +       u8 queue;
1636 +       u8 dest_type;
1637 +       u8 pad;
1638 +       __le64 user_ctx;
1639 +       union {
1640 +               __le32 options;
1641 +               __le32 fqid;
1642 +       };
1643 +};
1644 +
1645 +struct dpdmai_rsp_get_tx_queue {
1646 +       __le64 pad;
1647 +       __le32 fqid;
1648 +};
1649 +
1650 +
1651 +int dpdmai_open(struct fsl_mc_io *mc_io,
1652 +               uint32_t cmd_flags,
1653 +               int dpdmai_id,
1654 +               uint16_t *token)
1655 +{
1656 +       struct fsl_mc_command cmd = { 0 };
1657 +       struct dpdmai_cmd_open *cmd_params;
1658 +       int err;
1659 +
1660 +       /* prepare command */
1661 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN,
1662 +                                         cmd_flags,
1663 +                                         0);
1664 +
1665 +       cmd_params = (struct dpdmai_cmd_open *)cmd.params;
1666 +       cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id);
1667 +
1668 +       /* send command to mc*/
1669 +       err = mc_send_command(mc_io, &cmd);
1670 +       if (err)
1671 +               return err;
1672 +
1673 +       /* retrieve response parameters */
1674 +       *token = mc_cmd_hdr_read_token(&cmd);
1675 +       return 0;
1676 +}
1677 +
1678 +int dpdmai_close(struct fsl_mc_io *mc_io,
1679 +                uint32_t cmd_flags,
1680 +                uint16_t token)
1681 +{
1682 +       struct fsl_mc_command cmd = { 0 };
1683 +
1684 +       /* prepare command */
1685 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE,
1686 +                                         cmd_flags, token);
1687 +
1688 +       /* send command to mc*/
1689 +       return mc_send_command(mc_io, &cmd);
1690 +}
1691 +
1692 +int dpdmai_create(struct fsl_mc_io *mc_io,
1693 +                 uint32_t cmd_flags,
1694 +                 const struct dpdmai_cfg *cfg,
1695 +                 uint16_t *token)
1696 +{
1697 +       struct fsl_mc_command cmd = { 0 };
1698 +       int err;
1699 +
1700 +       /* prepare command */
1701 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE,
1702 +                                         cmd_flags,
1703 +                                         0);
1704 +       DPDMAI_CMD_CREATE(cmd, cfg);
1705 +
1706 +       /* send command to mc*/
1707 +       err = mc_send_command(mc_io, &cmd);
1708 +       if (err)
1709 +               return err;
1710 +
1711 +       /* retrieve response parameters */
1712 +       *token = MC_CMD_HDR_READ_TOKEN(cmd.header);
1713 +
1714 +       return 0;
1715 +}
1716 +
1717 +int dpdmai_destroy(struct fsl_mc_io *mc_io,
1718 +                  uint32_t cmd_flags,
1719 +                  uint16_t token)
1720 +{
1721 +       struct fsl_mc_command cmd = { 0 };
1722 +
1723 +       /* prepare command */
1724 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY,
1725 +                                         cmd_flags,
1726 +                                         token);
1727 +
1728 +       /* send command to mc*/
1729 +       return mc_send_command(mc_io, &cmd);
1730 +}
1731 +
1732 +int dpdmai_enable(struct fsl_mc_io *mc_io,
1733 +                 uint32_t cmd_flags,
1734 +                 uint16_t token)
1735 +{
1736 +       struct fsl_mc_command cmd = { 0 };
1737 +
1738 +       /* prepare command */
1739 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE,
1740 +                                         cmd_flags,
1741 +                                         token);
1742 +
1743 +       /* send command to mc*/
1744 +       return mc_send_command(mc_io, &cmd);
1745 +}
1746 +
1747 +int dpdmai_disable(struct fsl_mc_io *mc_io,
1748 +                  uint32_t cmd_flags,
1749 +                  uint16_t token)
1750 +{
1751 +       struct fsl_mc_command cmd = { 0 };
1752 +
1753 +       /* prepare command */
1754 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE,
1755 +                                         cmd_flags,
1756 +                                         token);
1757 +
1758 +       /* send command to mc*/
1759 +       return mc_send_command(mc_io, &cmd);
1760 +}
1761 +
1762 +int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
1763 +                     uint32_t cmd_flags,
1764 +                     uint16_t token,
1765 +                     int *en)
1766 +{
1767 +       struct fsl_mc_command cmd = { 0 };
1768 +       int err;
1769 +       /* prepare command */
1770 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_IS_ENABLED,
1771 +                                         cmd_flags,
1772 +                                         token);
1773 +
1774 +       /* send command to mc*/
1775 +       err = mc_send_command(mc_io, &cmd);
1776 +       if (err)
1777 +               return err;
1778 +
1779 +       /* retrieve response parameters */
1780 +       DPDMAI_RSP_IS_ENABLED(cmd, *en);
1781 +
1782 +       return 0;
1783 +}
1784 +
1785 +int dpdmai_reset(struct fsl_mc_io *mc_io,
1786 +                uint32_t cmd_flags,
1787 +                uint16_t token)
1788 +{
1789 +       struct fsl_mc_command cmd = { 0 };
1790 +
1791 +       /* prepare command */
1792 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET,
1793 +                                         cmd_flags,
1794 +                                         token);
1795 +
1796 +       /* send command to mc*/
1797 +       return mc_send_command(mc_io, &cmd);
1798 +}
1799 +
1800 +int dpdmai_get_irq(struct fsl_mc_io *mc_io,
1801 +                  uint32_t cmd_flags,
1802 +                  uint16_t token,
1803 +                  uint8_t irq_index,
1804 +                  int *type,
1805 +                  struct dpdmai_irq_cfg        *irq_cfg)
1806 +{
1807 +       struct fsl_mc_command cmd = { 0 };
1808 +       int err;
1809 +
1810 +       /* prepare command */
1811 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ,
1812 +                                         cmd_flags,
1813 +                                         token);
1814 +       DPDMAI_CMD_GET_IRQ(cmd, irq_index);
1815 +
1816 +       /* send command to mc*/
1817 +       err = mc_send_command(mc_io, &cmd);
1818 +       if (err)
1819 +               return err;
1820 +
1821 +       /* retrieve response parameters */
1822 +       DPDMAI_RSP_GET_IRQ(cmd, *type, irq_cfg);
1823 +
1824 +       return 0;
1825 +}
1826 +
1827 +int dpdmai_set_irq(struct fsl_mc_io *mc_io,
1828 +                  uint32_t cmd_flags,
1829 +                  uint16_t token,
1830 +                  uint8_t irq_index,
1831 +                  struct dpdmai_irq_cfg *irq_cfg)
1832 +{
1833 +       struct fsl_mc_command cmd = { 0 };
1834 +
1835 +       /* prepare command */
1836 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ,
1837 +                                         cmd_flags,
1838 +                                         token);
1839 +       DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg);
1840 +
1841 +       /* send command to mc*/
1842 +       return mc_send_command(mc_io, &cmd);
1843 +}
1844 +
1845 +int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io,
1846 +                         uint32_t cmd_flags,
1847 +                         uint16_t token,
1848 +                         uint8_t irq_index,
1849 +                         uint8_t *en)
1850 +{
1851 +       struct fsl_mc_command cmd = { 0 };
1852 +       int err;
1853 +
1854 +       /* prepare command */
1855 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_ENABLE,
1856 +                                         cmd_flags,
1857 +                                         token);
1858 +       DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index);
1859 +
1860 +       /* send command to mc*/
1861 +       err = mc_send_command(mc_io, &cmd);
1862 +       if (err)
1863 +               return err;
1864 +
1865 +       /* retrieve response parameters */
1866 +       DPDMAI_RSP_GET_IRQ_ENABLE(cmd, *en);
1867 +
1868 +       return 0;
1869 +}
1870 +
1871 +int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io,
1872 +                         uint32_t cmd_flags,
1873 +                         uint16_t token,
1874 +                         uint8_t irq_index,
1875 +                         uint8_t en)
1876 +{
1877 +       struct fsl_mc_command cmd = { 0 };
1878 +
1879 +       /* prepare command */
1880 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_ENABLE,
1881 +                                         cmd_flags,
1882 +                                         token);
1883 +       DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, en);
1884 +
1885 +       /* send command to mc*/
1886 +       return mc_send_command(mc_io, &cmd);
1887 +}
1888 +
1889 +int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io,
1890 +                       uint32_t cmd_flags,
1891 +                       uint16_t token,
1892 +                       uint8_t irq_index,
1893 +                       uint32_t *mask)
1894 +{
1895 +       struct fsl_mc_command cmd = { 0 };
1896 +       int err;
1897 +
1898 +       /* prepare command */
1899 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_MASK,
1900 +                                         cmd_flags,
1901 +                                         token);
1902 +       DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index);
1903 +
1904 +       /* send command to mc*/
1905 +       err = mc_send_command(mc_io, &cmd);
1906 +       if (err)
1907 +               return err;
1908 +
1909 +       /* retrieve response parameters */
1910 +       DPDMAI_RSP_GET_IRQ_MASK(cmd, *mask);
1911 +
1912 +       return 0;
1913 +}
1914 +
1915 +int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io,
1916 +                       uint32_t cmd_flags,
1917 +                       uint16_t token,
1918 +                       uint8_t irq_index,
1919 +                       uint32_t mask)
1920 +{
1921 +       struct fsl_mc_command cmd = { 0 };
1922 +
1923 +       /* prepare command */
1924 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_MASK,
1925 +                                         cmd_flags,
1926 +                                         token);
1927 +       DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask);
1928 +
1929 +       /* send command to mc*/
1930 +       return mc_send_command(mc_io, &cmd);
1931 +}
1932 +
1933 +int dpdmai_get_irq_status(struct fsl_mc_io *mc_io,
1934 +                         uint32_t cmd_flags,
1935 +                         uint16_t token,
1936 +                         uint8_t irq_index,
1937 +                         uint32_t *status)
1938 +{
1939 +       struct fsl_mc_command cmd = { 0 };
1940 +       int err;
1941 +
1942 +       /* prepare command */
1943 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_STATUS,
1944 +                                         cmd_flags,
1945 +                                         token);
1946 +       DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, *status);
1947 +
1948 +       /* send command to mc*/
1949 +       err = mc_send_command(mc_io, &cmd);
1950 +       if (err)
1951 +               return err;
1952 +
1953 +       /* retrieve response parameters */
1954 +       DPDMAI_RSP_GET_IRQ_STATUS(cmd, *status);
1955 +
1956 +       return 0;
1957 +}
1958 +
1959 +int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io,
1960 +                           uint32_t cmd_flags,
1961 +                           uint16_t token,
1962 +                           uint8_t irq_index,
1963 +                           uint32_t status)
1964 +{
1965 +       struct fsl_mc_command cmd = { 0 };
1966 +
1967 +       /* prepare command */
1968 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLEAR_IRQ_STATUS,
1969 +                                         cmd_flags,
1970 +                                         token);
1971 +       DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status);
1972 +
1973 +       /* send command to mc*/
1974 +       return mc_send_command(mc_io, &cmd);
1975 +}
1976 +
1977 +int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
1978 +                         uint32_t cmd_flags,
1979 +                         uint16_t token,
1980 +                         struct dpdmai_attr *attr)
1981 +{
1982 +       struct fsl_mc_command cmd = { 0 };
1983 +       int err;
1984 +       struct dpdmai_rsp_get_attributes *rsp_params;
1985 +
1986 +       /* prepare command */
1987 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR,
1988 +                                         cmd_flags,
1989 +                                         token);
1990 +
1991 +       /* send command to mc*/
1992 +       err = mc_send_command(mc_io, &cmd);
1993 +       if (err)
1994 +               return err;
1995 +
1996 +       /* retrieve response parameters */
1997 +       rsp_params = (struct dpdmai_rsp_get_attributes *)cmd.params;
1998 +       attr->id = le32_to_cpu(rsp_params->id);
1999 +       attr->version.major = le16_to_cpu(rsp_params->major);
2000 +       attr->version.minor = le16_to_cpu(rsp_params->minor);
2001 +       attr->num_of_priorities = rsp_params->num_of_priorities;
2002 +
2003 +
2004 +       return 0;
2005 +}
2006 +
2007 +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
2008 +                       uint32_t cmd_flags,
2009 +                       uint16_t token,
2010 +                       uint8_t priority,
2011 +                       const struct dpdmai_rx_queue_cfg *cfg)
2012 +{
2013 +       struct fsl_mc_command cmd = { 0 };
2014 +       struct dpdmai_cmd_queue *cmd_params;
2015 +
2016 +       /* prepare command */
2017 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE,
2018 +                                         cmd_flags,
2019 +                                         token);
2020 +
2021 +       cmd_params = (struct dpdmai_cmd_queue *)cmd.params;
2022 +       cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
2023 +       cmd_params->priority = cfg->dest_cfg.priority;
2024 +       cmd_params->queue = priority;
2025 +       cmd_params->dest_type = cfg->dest_cfg.dest_type;
2026 +       cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx);
2027 +       cmd_params->options = cpu_to_le32(cfg->options);
2028 +
2029 +
2030 +       /* send command to mc*/
2031 +       return mc_send_command(mc_io, &cmd);
2032 +}
2033 +
2034 +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
2035 +                       uint32_t cmd_flags,
2036 +                       uint16_t token,
2037 +                       uint8_t priority, struct dpdmai_rx_queue_attr *attr)
2038 +{
2039 +       struct fsl_mc_command cmd = { 0 };
2040 +       struct dpdmai_cmd_queue *cmd_params;
2041 +       int err;
2042 +
2043 +       /* prepare command */
2044 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE,
2045 +                                         cmd_flags,
2046 +                                         token);
2047 +
2048 +       cmd_params = (struct dpdmai_cmd_queue *)cmd.params;
2049 +       cmd_params->queue = priority;
2050 +
2051 +       /* send command to mc*/
2052 +       err = mc_send_command(mc_io, &cmd);
2053 +       if (err)
2054 +               return err;
2055 +
2056 +       /* retrieve response parameters */
2057 +       attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id);
2058 +       attr->dest_cfg.priority = cmd_params->priority;
2059 +       attr->dest_cfg.dest_type = cmd_params->dest_type;
2060 +       attr->user_ctx = le64_to_cpu(cmd_params->user_ctx);
2061 +       attr->fqid = le32_to_cpu(cmd_params->fqid);
2062 +
2063 +       return 0;
2064 +}
2065 +
2066 +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
2067 +                       uint32_t cmd_flags,
2068 +                       uint16_t token,
2069 +                       uint8_t priority,
2070 +                       struct dpdmai_tx_queue_attr *attr)
2071 +{
2072 +       struct fsl_mc_command cmd = { 0 };
2073 +       struct dpdmai_cmd_queue *cmd_params;
2074 +       struct dpdmai_rsp_get_tx_queue *rsp_params;
2075 +       int err;
2076 +
2077 +       /* prepare command */
2078 +       cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE,
2079 +                                         cmd_flags,
2080 +                                         token);
2081 +
2082 +       cmd_params = (struct dpdmai_cmd_queue *)cmd.params;
2083 +       cmd_params->queue = priority;
2084 +
2085 +       /* send command to mc*/
2086 +       err = mc_send_command(mc_io, &cmd);
2087 +       if (err)
2088 +               return err;
2089 +
2090 +       /* retrieve response parameters */
2091 +
2092 +       rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params;
2093 +       attr->fqid = le32_to_cpu(rsp_params->fqid);
2094 +
2095 +       return 0;
2096 +}
2097 --- /dev/null
2098 +++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai.h
2099 @@ -0,0 +1,521 @@
2100 +/* Copyright 2013-2015 Freescale Semiconductor Inc.
2101 + *
2102 + * Redistribution and use in source and binary forms, with or without
2103 + * modification, are permitted provided that the following conditions are met:
2104 + * * Redistributions of source code must retain the above copyright
2105 + * notice, this list of conditions and the following disclaimer.
2106 + * * Redistributions in binary form must reproduce the above copyright
2107 + * notice, this list of conditions and the following disclaimer in the
2108 + * documentation and/or other materials provided with the distribution.
2109 + * * Neither the name of the above-listed copyright holders nor the
2110 + * names of any contributors may be used to endorse or promote products
2111 + * derived from this software without specific prior written permission.
2112 + *
2113 + *
2114 + * ALTERNATIVELY, this software may be distributed under the terms of the
2115 + * GNU General Public License ("GPL") as published by the Free Software
2116 + * Foundation, either version 2 of that License or (at your option) any
2117 + * later version.
2118 + *
2119 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
2120 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2121 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2122 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
2123 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
2124 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
2125 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
2126 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
2127 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
2128 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
2129 + * POSSIBILITY OF SUCH DAMAGE.
2130 + */
2131 +#ifndef __FSL_DPDMAI_H
2132 +#define __FSL_DPDMAI_H
2133 +
2134 +struct fsl_mc_io;
2135 +
2136 +/* Data Path DMA Interface API
2137 + * Contains initialization APIs and runtime control APIs for DPDMAI
2138 + */
2139 +
2140 +/* General DPDMAI macros */
2141 +
2142 +/**
2143 + * Maximum number of Tx/Rx priorities per DPDMAI object
2144 + */
2145 +#define DPDMAI_PRIO_NUM                2
2146 +
2147 +/**
2148 + * All queues considered; see dpdmai_set_rx_queue()
2149 + */
2150 +#define DPDMAI_ALL_QUEUES      (uint8_t)(-1)
2151 +
2152 +/**
2153 + * dpdmai_open() - Open a control session for the specified object
2154 + * @mc_io:     Pointer to MC portal's I/O object
2155 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2156 + * @dpdmai_id: DPDMAI unique ID
2157 + * @token:     Returned token; use in subsequent API calls
2158 + *
2159 + * This function can be used to open a control session for an
2160 + * already created object; an object may have been declared in
2161 + * the DPL or by calling the dpdmai_create() function.
2162 + * This function returns a unique authentication token,
2163 + * associated with the specific object ID and the specific MC
2164 + * portal; this token must be used in all subsequent commands for
2165 + * this specific object.
2166 + *
2167 + * Return:     '0' on Success; Error code otherwise.
2168 + */
2169 +int dpdmai_open(struct fsl_mc_io       *mc_io,
2170 +               uint32_t                cmd_flags,
2171 +               int                     dpdmai_id,
2172 +               uint16_t                *token);
2173 +
2174 +/**
2175 + * dpdmai_close() - Close the control session of the object
2176 + * @mc_io:     Pointer to MC portal's I/O object
2177 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2178 + * @token:     Token of DPDMAI object
2179 + *
2180 + * After this function is called, no further operations are
2181 + * allowed on the object without opening a new control session.
2182 + *
2183 + * Return:     '0' on Success; Error code otherwise.
2184 + */
2185 +int dpdmai_close(struct fsl_mc_io      *mc_io,
2186 +                uint32_t               cmd_flags,
2187 +                uint16_t               token);
2188 +
2189 +/**
2190 + * struct dpdmai_cfg - Structure representing DPDMAI configuration
2191 + * @priorities: Priorities for the DMA hardware processing; valid priorities are
2192 + *     configured with values 1-8; the entry following last valid entry
2193 + *     should be configured with 0
2194 + */
2195 +struct dpdmai_cfg {
2196 +       uint8_t priorities[DPDMAI_PRIO_NUM];
2197 +};
2198 +
2199 +/**
2200 + * dpdmai_create() - Create the DPDMAI object
2201 + * @mc_io:     Pointer to MC portal's I/O object
2202 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2203 + * @cfg:       Configuration structure
2204 + * @token:     Returned token; use in subsequent API calls
2205 + *
2206 + * Create the DPDMAI object, allocate required resources and
2207 + * perform required initialization.
2208 + *
2209 + * The object can be created either by declaring it in the
2210 + * DPL file, or by calling this function.
2211 + *
2212 + * This function returns a unique authentication token,
2213 + * associated with the specific object ID and the specific MC
2214 + * portal; this token must be used in all subsequent calls to
2215 + * this specific object. For objects that are created using the
2216 + * DPL file, call dpdmai_open() function to get an authentication
2217 + * token first.
2218 + *
2219 + * Return:     '0' on Success; Error code otherwise.
2220 + */
2221 +int dpdmai_create(struct fsl_mc_io             *mc_io,
2222 +                 uint32_t                      cmd_flags,
2223 +                 const struct dpdmai_cfg       *cfg,
2224 +                 uint16_t                      *token);
2225 +
2226 +/**
2227 + * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources.
2228 + * @mc_io:     Pointer to MC portal's I/O object
2229 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2230 + * @token:     Token of DPDMAI object
2231 + *
2232 + * Return:     '0' on Success; error code otherwise.
2233 + */
2234 +int dpdmai_destroy(struct fsl_mc_io    *mc_io,
2235 +                  uint32_t             cmd_flags,
2236 +                  uint16_t             token);
2237 +
2238 +/**
2239 + * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames.
2240 + * @mc_io:     Pointer to MC portal's I/O object
2241 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2242 + * @token:     Token of DPDMAI object
2243 + *
2244 + * Return:     '0' on Success; Error code otherwise.
2245 + */
2246 +int dpdmai_enable(struct fsl_mc_io     *mc_io,
2247 +                 uint32_t              cmd_flags,
2248 +                 uint16_t              token);
2249 +
2250 +/**
2251 + * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames.
2252 + * @mc_io:     Pointer to MC portal's I/O object
2253 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2254 + * @token:     Token of DPDMAI object
2255 + *
2256 + * Return:     '0' on Success; Error code otherwise.
2257 + */
2258 +int dpdmai_disable(struct fsl_mc_io    *mc_io,
2259 +                  uint32_t             cmd_flags,
2260 +                  uint16_t             token);
2261 +
2262 +/**
2263 + * dpdmai_is_enabled() - Check if the DPDMAI is enabled.
2264 + * @mc_io:     Pointer to MC portal's I/O object
2265 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2266 + * @token:     Token of DPDMAI object
2267 + * @en:                Returns '1' if object is enabled; '0' otherwise
2268 + *
2269 + * Return:     '0' on Success; Error code otherwise.
2270 + */
2271 +int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
2272 +                     uint32_t          cmd_flags,
2273 +                     uint16_t          token,
2274 +                     int               *en);
2275 +
2276 +/**
2277 + * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state.
2278 + * @mc_io:     Pointer to MC portal's I/O object
2279 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2280 + * @token:     Token of DPDMAI object
2281 + *
2282 + * Return:     '0' on Success; Error code otherwise.
2283 + */
2284 +int dpdmai_reset(struct fsl_mc_io      *mc_io,
2285 +                uint32_t               cmd_flags,
2286 +                uint16_t               token);
2287 +
2288 +/**
2289 + * struct dpdmai_irq_cfg - IRQ configuration
2290 + * @addr:      Address that must be written to signal a message-based interrupt
2291 + * @val:       Value to write into irq_addr address
2292 + * @irq_num: A user defined number associated with this IRQ
2293 + */
2294 +struct dpdmai_irq_cfg {
2295 +            uint64_t           addr;
2296 +            uint32_t           val;
2297 +            int                irq_num;
2298 +};
2299 +
2300 +/**
2301 + * dpdmai_set_irq() - Set IRQ information for the DPDMAI to trigger an interrupt.
2302 + * @mc_io:     Pointer to MC portal's I/O object
2303 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2304 + * @token:     Token of DPDMAI object
2305 + * @irq_index: Identifies the interrupt index to configure
2306 + * @irq_cfg:   IRQ configuration
2307 + *
2308 + * Return:     '0' on Success; Error code otherwise.
2309 + */
2310 +int dpdmai_set_irq(struct fsl_mc_io            *mc_io,
2311 +                  uint32_t                     cmd_flags,
2312 +                  uint16_t                     token,
2313 +                  uint8_t                      irq_index,
2314 +                  struct dpdmai_irq_cfg        *irq_cfg);
2315 +
2316 +/**
2317 + * dpdmai_get_irq() - Get IRQ information from the DPDMAI
2318 + *
2319 + * @mc_io:     Pointer to MC portal's I/O object
2320 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2321 + * @token:     Token of DPDMAI object
2322 + * @irq_index: The interrupt index to configure
2323 + * @type:      Interrupt type: 0 represents message interrupt
2324 + *             type (both irq_addr and irq_val are valid)
2325 + * @irq_cfg:   IRQ attributes
2326 + *
2327 + * Return:     '0' on Success; Error code otherwise.
2328 + */
2329 +int dpdmai_get_irq(struct fsl_mc_io            *mc_io,
2330 +                  uint32_t                     cmd_flags,
2331 +                  uint16_t                     token,
2332 +                  uint8_t                      irq_index,
2333 +                  int                          *type,
2334 +                  struct dpdmai_irq_cfg        *irq_cfg);
2335 +
2336 +/**
2337 + * dpdmai_set_irq_enable() - Set overall interrupt state.
2338 + * @mc_io:     Pointer to MC portal's I/O object
2339 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2340 + * @token:             Token of DPDMAI object
2341 + * @irq_index: The interrupt index to configure
2342 + * @en:                        Interrupt state - enable = 1, disable = 0
2343 + *
2344 + * Allows GPP software to control when interrupts are generated.
2345 + * Each interrupt can have up to 32 causes.  The enable/disable control's the
2346 + * overall interrupt state. if the interrupt is disabled no causes will cause
2347 + * an interrupt
2348 + *
2349 + * Return:     '0' on Success; Error code otherwise.
2350 + */
2351 +int dpdmai_set_irq_enable(struct fsl_mc_io     *mc_io,
2352 +                         uint32_t              cmd_flags,
2353 +                         uint16_t              token,
2354 +                         uint8_t               irq_index,
2355 +                         uint8_t               en);
2356 +
2357 +/**
2358 + * dpdmai_get_irq_enable() - Get overall interrupt state
2359 + * @mc_io:     Pointer to MC portal's I/O object
2360 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2361 + * @token:             Token of DPDMAI object
2362 + * @irq_index: The interrupt index to configure
2363 + * @en:                        Returned Interrupt state - enable = 1, disable = 0
2364 + *
2365 + * Return:     '0' on Success; Error code otherwise.
2366 + */
2367 +int dpdmai_get_irq_enable(struct fsl_mc_io     *mc_io,
2368 +                         uint32_t              cmd_flags,
2369 +                         uint16_t              token,
2370 +                         uint8_t               irq_index,
2371 +                         uint8_t               *en);
2372 +
2373 +/**
2374 + * dpdmai_set_irq_mask() - Set interrupt mask.
2375 + * @mc_io:     Pointer to MC portal's I/O object
2376 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2377 + * @token:             Token of DPDMAI object
2378 + * @irq_index: The interrupt index to configure
2379 + * @mask:              event mask to trigger interrupt;
2380 + *                             each bit:
2381 + *                                     0 = ignore event
2382 + *                                     1 = consider event for asserting IRQ
2383 + *
2384 + * Every interrupt can have up to 32 causes and the interrupt model supports
2385 + * masking/unmasking each cause independently
2386 + *
2387 + * Return:     '0' on Success; Error code otherwise.
2388 + */
2389 +int dpdmai_set_irq_mask(struct fsl_mc_io       *mc_io,
2390 +                       uint32_t                cmd_flags,
2391 +                       uint16_t                token,
2392 +                       uint8_t         irq_index,
2393 +                       uint32_t                mask);
2394 +
2395 +/**
2396 + * dpdmai_get_irq_mask() - Get interrupt mask.
2397 + * @mc_io:     Pointer to MC portal's I/O object
2398 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2399 + * @token:             Token of DPDMAI object
2400 + * @irq_index: The interrupt index to configure
2401 + * @mask:              Returned event mask to trigger interrupt
2402 + *
2403 + * Every interrupt can have up to 32 causes and the interrupt model supports
2404 + * masking/unmasking each cause independently
2405 + *
2406 + * Return:     '0' on Success; Error code otherwise.
2407 + */
2408 +int dpdmai_get_irq_mask(struct fsl_mc_io       *mc_io,
2409 +                       uint32_t                cmd_flags,
2410 +                       uint16_t                token,
2411 +                       uint8_t         irq_index,
2412 +                       uint32_t                *mask);
2413 +
2414 +/**
2415 + * dpdmai_get_irq_status() - Get the current status of any pending interrupts
2416 + * @mc_io:     Pointer to MC portal's I/O object
2417 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2418 + * @token:             Token of DPDMAI object
2419 + * @irq_index: The interrupt index to configure
2420 + * @status:            Returned interrupts status - one bit per cause:
2421 + *                                     0 = no interrupt pending
2422 + *                                     1 = interrupt pending
2423 + *
2424 + * Return:     '0' on Success; Error code otherwise.
2425 + */
2426 +int dpdmai_get_irq_status(struct fsl_mc_io     *mc_io,
2427 +                         uint32_t              cmd_flags,
2428 +                         uint16_t              token,
2429 +                         uint8_t               irq_index,
2430 +                         uint32_t              *status);
2431 +
2432 +/**
2433 + * dpdmai_clear_irq_status() - Clear a pending interrupt's status
2434 + * @mc_io:     Pointer to MC portal's I/O object
2435 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2436 + * @token:     Token of DPDMAI object
2437 + * @irq_index: The interrupt index to configure
2438 + * @status:    bits to clear (W1C) - one bit per cause:
2439 + *                     0 = don't change
2440 + *                     1 = clear status bit
2441 + *
2442 + * Return:     '0' on Success; Error code otherwise.
2443 + */
2444 +int dpdmai_clear_irq_status(struct fsl_mc_io   *mc_io,
2445 +                           uint32_t            cmd_flags,
2446 +                           uint16_t            token,
2447 +                           uint8_t             irq_index,
2448 +                           uint32_t            status);
2449 +
2450 +/**
2451 + * struct dpdmai_attr - Structure representing DPDMAI attributes
2452 + * @id: DPDMAI object ID
2453 + * @version: DPDMAI version
2454 + * @num_of_priorities: number of priorities
2455 + */
2456 +struct dpdmai_attr {
2457 +       int     id;
2458 +       /**
2459 +        * struct version - DPDMAI version
2460 +        * @major: DPDMAI major version
2461 +        * @minor: DPDMAI minor version
2462 +        */
2463 +       struct {
2464 +               uint16_t major;
2465 +               uint16_t minor;
2466 +       } version;
2467 +       uint8_t num_of_priorities;
2468 +};
2469 +
2470 +/**
2471 + * dpdmai_get_attributes() - Retrieve DPDMAI attributes.
2472 + * @mc_io:     Pointer to MC portal's I/O object
2473 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2474 + * @token:     Token of DPDMAI object
2475 + * @attr:      Returned object's attributes
2476 + *
2477 + * Return:     '0' on Success; Error code otherwise.
2478 + */
2479 +int dpdmai_get_attributes(struct fsl_mc_io     *mc_io,
2480 +                         uint32_t              cmd_flags,
2481 +                         uint16_t              token,
2482 +                         struct dpdmai_attr    *attr);
2483 +
2484 +/**
2485 + * enum dpdmai_dest - DPDMAI destination types
2486 + * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode
2487 + *     and does not generate FQDAN notifications; user is expected to dequeue
2488 + *     from the queue based on polling or other user-defined method
2489 + * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN
2490 + *     notifications to the specified DPIO; user is expected to dequeue
2491 + *     from the queue only after notification is received
2492 + * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate
2493 + *     FQDAN notifications, but is connected to the specified DPCON object;
2494 + *     user is expected to dequeue from the DPCON channel
2495 + */
2496 +enum dpdmai_dest {
2497 +       DPDMAI_DEST_NONE = 0,
2498 +       DPDMAI_DEST_DPIO = 1,
2499 +       DPDMAI_DEST_DPCON = 2
2500 +};
2501 +
2502 +/**
2503 + * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters
2504 + * @dest_type: Destination type
2505 + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type
2506 + * @priority: Priority selection within the DPIO or DPCON channel; valid values
2507 + *     are 0-1 or 0-7, depending on the number of priorities in that
2508 + *     channel; not relevant for 'DPDMAI_DEST_NONE' option
2509 + */
2510 +struct dpdmai_dest_cfg {
2511 +       enum dpdmai_dest        dest_type;
2512 +       int                     dest_id;
2513 +       uint8_t         priority;
2514 +};
2515 +
2516 +/* DPDMAI queue modification options */
2517 +
2518 +/**
2519 + * Select to modify the user's context associated with the queue
2520 + */
2521 +#define DPDMAI_QUEUE_OPT_USER_CTX      0x00000001
2522 +
2523 +/**
2524 + * Select to modify the queue's destination
2525 + */
2526 +#define DPDMAI_QUEUE_OPT_DEST          0x00000002
2527 +
2528 +/**
2529 + * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration
2530 + * @options: Flags representing the suggested modifications to the queue;
2531 + *     Use any combination of 'DPDMAI_QUEUE_OPT_<X>' flags
2532 + * @user_ctx: User context value provided in the frame descriptor of each
2533 + *     dequeued frame;
2534 + *     valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options'
2535 + * @dest_cfg: Queue destination parameters;
2536 + *     valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options'
2537 + */
2538 +struct dpdmai_rx_queue_cfg {
2539 +       uint32_t                options;
2540 +       uint64_t                user_ctx;
2541 +       struct dpdmai_dest_cfg  dest_cfg;
2542 +
2543 +};
2544 +
2545 +/**
2546 + * dpdmai_set_rx_queue() - Set Rx queue configuration
2547 + * @mc_io:     Pointer to MC portal's I/O object
2548 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2549 + * @token:     Token of DPDMAI object
2550 + * @priority:  Select the queue relative to number of
2551 + *                     priorities configured at DPDMAI creation; use
2552 + *                     DPDMAI_ALL_QUEUES to configure all Rx queues
2553 + *                     identically.
2554 + * @cfg:       Rx queue configuration
2555 + *
2556 + * Return:     '0' on Success; Error code otherwise.
2557 + */
2558 +int dpdmai_set_rx_queue(struct fsl_mc_io                       *mc_io,
2559 +                       uint32_t                                cmd_flags,
2560 +                       uint16_t                                token,
2561 +                       uint8_t                                 priority,
2562 +                       const struct dpdmai_rx_queue_cfg        *cfg);
2563 +
2564 +/**
2565 + * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues
2566 + * @user_ctx:  User context value provided in the frame descriptor of each
2567 + *      dequeued frame
2568 + * @dest_cfg: Queue destination configuration
2569 + * @fqid: Virtual FQID value to be used for dequeue operations
2570 + */
2571 +struct dpdmai_rx_queue_attr {
2572 +       uint64_t                user_ctx;
2573 +       struct dpdmai_dest_cfg  dest_cfg;
2574 +       uint32_t                fqid;
2575 +};
2576 +
2577 +/**
2578 + * dpdmai_get_rx_queue() - Retrieve Rx queue attributes.
2579 + * @mc_io:     Pointer to MC portal's I/O object
2580 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2581 + * @token:     Token of DPDMAI object
2582 + * @priority:  Select the queue relative to number of
2583 + *                             priorities configured at DPDMAI creation
2584 + * @attr:      Returned Rx queue attributes
2585 + *
2586 + * Return:     '0' on Success; Error code otherwise.
2587 + */
2588 +int dpdmai_get_rx_queue(struct fsl_mc_io               *mc_io,
2589 +                       uint32_t                        cmd_flags,
2590 +                       uint16_t                        token,
2591 +                       uint8_t                         priority,
2592 +                       struct dpdmai_rx_queue_attr     *attr);
2593 +
2594 +/**
2595 + * struct dpdmai_tx_queue_attr - Structure representing attributes of Tx queues
2596 + * @fqid: Virtual FQID to be used for sending frames to DMA hardware
2597 + */
2598 +
2599 +struct dpdmai_tx_queue_attr {
2600 +       uint32_t fqid;
2601 +};
2602 +
2603 +/**
2604 + * dpdmai_get_tx_queue() - Retrieve Tx queue attributes.
2605 + * @mc_io:     Pointer to MC portal's I/O object
2606 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2607 + * @token:     Token of DPDMAI object
2608 + * @priority:  Select the queue relative to number of
2609 + *                     priorities configured at DPDMAI creation
2610 + * @attr:      Returned Tx queue attributes
2611 + *
2612 + * Return:     '0' on Success; Error code otherwise.
2613 + */
2614 +int dpdmai_get_tx_queue(struct fsl_mc_io               *mc_io,
2615 +                       uint32_t                        cmd_flags,
2616 +                       uint16_t                        token,
2617 +                       uint8_t                         priority,
2618 +                       struct dpdmai_tx_queue_attr     *attr);
2619 +
2620 +#endif /* __FSL_DPDMAI_H */
2621 --- /dev/null
2622 +++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h
2623 @@ -0,0 +1,222 @@
2624 +/* Copyright 2013-2016 Freescale Semiconductor Inc.
2625 + *
2626 + * Redistribution and use in source and binary forms, with or without
2627 + * modification, are permitted provided that the following conditions are met:
2628 + * * Redistributions of source code must retain the above copyright
2629 + * notice, this list of conditions and the following disclaimer.
2630 + * * Redistributions in binary form must reproduce the above copyright
2631 + * notice, this list of conditions and the following disclaimer in the
2632 + * documentation and/or other materials provided with the distribution.
2633 + * * Neither the name of the above-listed copyright holders nor the
2634 + * names of any contributors may be used to endorse or promote products
2635 + * derived from this software without specific prior written permission.
2636 + *
2637 + *
2638 + * ALTERNATIVELY, this software may be distributed under the terms of the
2639 + * GNU General Public License ("GPL") as published by the Free Software
2640 + * Foundation, either version 2 of that License or (at your option) any
2641 + * later version.
2642 + *
2643 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
2644 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2645 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2646 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
2647 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
2648 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
2649 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
2650 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
2651 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
2652 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
2653 + * POSSIBILITY OF SUCH DAMAGE.
2654 + */
2655 +#ifndef _FSL_DPDMAI_CMD_H
2656 +#define _FSL_DPDMAI_CMD_H
2657 +
2658 +/* DPDMAI Version */
2659 +#define DPDMAI_VER_MAJOR                               2
2660 +#define DPDMAI_VER_MINOR                               2
2661 +
2662 +#define DPDMAI_CMD_BASE_VERSION                        0
2663 +#define DPDMAI_CMD_ID_OFFSET                           4
2664 +
2665 +/* Command IDs */
2666 +#define DPDMAI_CMDID_CLOSE                           ((0x800 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2667 +#define DPDMAI_CMDID_OPEN                            ((0x80E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2668 +#define DPDMAI_CMDID_CREATE                          ((0x90E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2669 +#define DPDMAI_CMDID_DESTROY                         ((0x900 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2670 +
2671 +#define DPDMAI_CMDID_ENABLE                          ((0x002 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2672 +#define DPDMAI_CMDID_DISABLE                         ((0x003 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2673 +#define DPDMAI_CMDID_GET_ATTR                        ((0x004 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2674 +#define DPDMAI_CMDID_RESET                           ((0x005 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2675 +#define DPDMAI_CMDID_IS_ENABLED                      ((0x006 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2676 +
2677 +#define DPDMAI_CMDID_SET_IRQ                         ((0x010 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2678 +#define DPDMAI_CMDID_GET_IRQ                         ((0x011 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2679 +#define DPDMAI_CMDID_SET_IRQ_ENABLE                  ((0x012 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2680 +#define DPDMAI_CMDID_GET_IRQ_ENABLE                  ((0x013 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2681 +#define DPDMAI_CMDID_SET_IRQ_MASK                    ((0x014 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2682 +#define DPDMAI_CMDID_GET_IRQ_MASK                    ((0x015 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2683 +#define DPDMAI_CMDID_GET_IRQ_STATUS                  ((0x016 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2684 +#define DPDMAI_CMDID_CLEAR_IRQ_STATUS                ((0x017 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2685 +
2686 +#define DPDMAI_CMDID_SET_RX_QUEUE                    ((0x1A0 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2687 +#define DPDMAI_CMDID_GET_RX_QUEUE                    ((0x1A1 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2688 +#define DPDMAI_CMDID_GET_TX_QUEUE                    ((0x1A2 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2689 +
2690 +
2691 +#define MC_CMD_HDR_TOKEN_O 32  /* Token field offset */
2692 +#define MC_CMD_HDR_TOKEN_S 16  /* Token field size */
2693 +
2694 +
2695 +#define MAKE_UMASK64(_width) \
2696 +       ((uint64_t)((_width) < 64 ? ((uint64_t)1 << (_width)) - 1 : \
2697 +       (uint64_t)-1))
2698 +
2699 +static inline uint64_t mc_enc(int lsoffset, int width, uint64_t val)
2700 +{
2701 +       return (uint64_t)(((uint64_t)val & MAKE_UMASK64(width)) << lsoffset);
2702 +}
2703 +
2704 +static inline uint64_t mc_dec(uint64_t val, int lsoffset, int width)
2705 +{
2706 +       return (uint64_t)((val >> lsoffset) & MAKE_UMASK64(width));
2707 +}
2708 +
2709 +#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \
2710 +       ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg))
2711 +
2712 +#define MC_RSP_OP(_cmd, _param, _offset, _width, _type, _arg) \
2713 +       (_arg = (_type)mc_dec(_cmd.params[_param], (_offset), (_width)))
2714 +
2715 +#define MC_CMD_HDR_READ_TOKEN(_hdr) \
2716 +       ((uint16_t)mc_dec((_hdr), MC_CMD_HDR_TOKEN_O, MC_CMD_HDR_TOKEN_S))
2717 +
2718 +/*                cmd, param, offset, width, type, arg_name */
2719 +#define DPDMAI_CMD_OPEN(cmd, dpdmai_id) \
2720 +       MC_CMD_OP(cmd, 0, 0,  32, int,      dpdmai_id)
2721 +
2722 +/*                cmd, param, offset, width, type, arg_name */
2723 +#define DPDMAI_CMD_CREATE(cmd, cfg) \
2724 +do { \
2725 +       MC_CMD_OP(cmd, 0, 8,  8,  uint8_t,  cfg->priorities[0]);\
2726 +       MC_CMD_OP(cmd, 0, 16, 8,  uint8_t,  cfg->priorities[1]);\
2727 +} while (0)
2728 +
2729 +/*                cmd, param, offset, width, type, arg_name */
2730 +#define DPDMAI_RSP_IS_ENABLED(cmd, en) \
2731 +       MC_RSP_OP(cmd, 0, 0,  1,  int,      en)
2732 +
2733 +/*                cmd, param, offset, width, type, arg_name */
2734 +#define DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg) \
2735 +do { \
2736 +       MC_CMD_OP(cmd, 0, 0,  8,  uint8_t,  irq_index);\
2737 +       MC_CMD_OP(cmd, 0, 32, 32, uint32_t, irq_cfg->val);\
2738 +       MC_CMD_OP(cmd, 1, 0,  64, uint64_t, irq_cfg->addr);\
2739 +       MC_CMD_OP(cmd, 2, 0,  32, int,      irq_cfg->irq_num); \
2740 +} while (0)
2741 +
2742 +/*                cmd, param, offset, width, type, arg_name */
2743 +#define DPDMAI_CMD_GET_IRQ(cmd, irq_index) \
2744 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  irq_index)
2745 +
2746 +/*                cmd, param, offset, width, type, arg_name */
2747 +#define DPDMAI_RSP_GET_IRQ(cmd, type, irq_cfg) \
2748 +do { \
2749 +       MC_RSP_OP(cmd, 0, 0,  32, uint32_t, irq_cfg->val); \
2750 +       MC_RSP_OP(cmd, 1, 0,  64, uint64_t, irq_cfg->addr);\
2751 +       MC_RSP_OP(cmd, 2, 0,  32, int,      irq_cfg->irq_num); \
2752 +       MC_RSP_OP(cmd, 2, 32, 32, int,      type); \
2753 +} while (0)
2754 +
2755 +/*                cmd, param, offset, width, type, arg_name */
2756 +#define DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, enable_state) \
2757 +do { \
2758 +       MC_CMD_OP(cmd, 0, 0,  8,  uint8_t,  enable_state); \
2759 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  irq_index); \
2760 +} while (0)
2761 +
2762 +/*                cmd, param, offset, width, type, arg_name */
2763 +#define DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index) \
2764 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  irq_index)
2765 +
2766 +/*                cmd, param, offset, width, type, arg_name */
2767 +#define DPDMAI_RSP_GET_IRQ_ENABLE(cmd, enable_state) \
2768 +       MC_RSP_OP(cmd, 0, 0,  8,  uint8_t,  enable_state)
2769 +
2770 +/*                cmd, param, offset, width, type, arg_name */
2771 +#define DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask) \
2772 +do { \
2773 +       MC_CMD_OP(cmd, 0, 0,  32, uint32_t, mask); \
2774 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  irq_index); \
2775 +} while (0)
2776 +
2777 +/*                cmd, param, offset, width, type, arg_name */
2778 +#define DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index) \
2779 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  irq_index)
2780 +
2781 +/*                cmd, param, offset, width, type, arg_name */
2782 +#define DPDMAI_RSP_GET_IRQ_MASK(cmd, mask) \
2783 +       MC_RSP_OP(cmd, 0, 0,  32, uint32_t, mask)
2784 +
2785 +/*                cmd, param, offset, width, type, arg_name */
2786 +#define DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, status) \
2787 +do { \
2788 +       MC_CMD_OP(cmd, 0, 0,  32, uint32_t, status);\
2789 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  irq_index);\
2790 +} while (0)
2791 +
2792 +/*                cmd, param, offset, width, type, arg_name */
2793 +#define DPDMAI_RSP_GET_IRQ_STATUS(cmd, status) \
2794 +       MC_RSP_OP(cmd, 0, 0,  32, uint32_t,  status)
2795 +
2796 +/*                cmd, param, offset, width, type, arg_name */
2797 +#define DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status) \
2798 +do { \
2799 +       MC_CMD_OP(cmd, 0, 0,  32, uint32_t, status); \
2800 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  irq_index); \
2801 +} while (0)
2802 +
2803 +/*                cmd, param, offset, width, type, arg_name */
2804 +#define DPDMAI_RSP_GET_ATTR(cmd, attr) \
2805 +do { \
2806 +       MC_RSP_OP(cmd, 0, 0,  32, int,      attr->id); \
2807 +       MC_RSP_OP(cmd, 0, 32,  8,  uint8_t,  attr->num_of_priorities); \
2808 +       MC_RSP_OP(cmd, 1, 0,  16, uint16_t, attr->version.major);\
2809 +       MC_RSP_OP(cmd, 1, 16, 16, uint16_t, attr->version.minor);\
2810 +} while (0)
2811 +
2812 +/*                cmd, param, offset, width, type, arg_name */
2813 +#define DPDMAI_CMD_SET_RX_QUEUE(cmd, priority, cfg) \
2814 +do { \
2815 +       MC_CMD_OP(cmd, 0, 0,  32, int,      cfg->dest_cfg.dest_id); \
2816 +       MC_CMD_OP(cmd, 0, 32, 8,  uint8_t,  cfg->dest_cfg.priority); \
2817 +       MC_CMD_OP(cmd, 0, 40, 8,  uint8_t,  priority); \
2818 +       MC_CMD_OP(cmd, 0, 48, 4,  enum dpdmai_dest, cfg->dest_cfg.dest_type); \
2819 +       MC_CMD_OP(cmd, 1, 0,  64, uint64_t, cfg->user_ctx); \
2820 +       MC_CMD_OP(cmd, 2, 0,  32, uint32_t, cfg->options);\
2821 +} while (0)
2822 +
2823 +/*                cmd, param, offset, width, type, arg_name */
2824 +#define DPDMAI_CMD_GET_RX_QUEUE(cmd, priority) \
2825 +       MC_CMD_OP(cmd, 0, 40, 8,  uint8_t,  priority)
2826 +
2827 +/*                cmd, param, offset, width, type, arg_name */
2828 +#define DPDMAI_RSP_GET_RX_QUEUE(cmd, attr) \
2829 +do { \
2830 +       MC_RSP_OP(cmd, 0, 0,  32, int,      attr->dest_cfg.dest_id);\
2831 +       MC_RSP_OP(cmd, 0, 32, 8,  uint8_t,  attr->dest_cfg.priority);\
2832 +       MC_RSP_OP(cmd, 0, 48, 4,  enum dpdmai_dest, attr->dest_cfg.dest_type);\
2833 +       MC_RSP_OP(cmd, 1, 0,  64, uint64_t,  attr->user_ctx);\
2834 +       MC_RSP_OP(cmd, 2, 0,  32, uint32_t,  attr->fqid);\
2835 +} while (0)
2836 +
2837 +/*                cmd, param, offset, width, type, arg_name */
2838 +#define DPDMAI_CMD_GET_TX_QUEUE(cmd, priority) \
2839 +       MC_CMD_OP(cmd, 0, 40, 8,  uint8_t,  priority)
2840 +
2841 +/*                cmd, param, offset, width, type, arg_name */
2842 +#define DPDMAI_RSP_GET_TX_QUEUE(cmd, attr) \
2843 +       MC_RSP_OP(cmd, 1, 0,  32, uint32_t,  attr->fqid)
2844 +
2845 +#endif /* _FSL_DPDMAI_CMD_H */
2846 --- a/drivers/dma/fsl-edma.c
2847 +++ b/drivers/dma/fsl-edma.c
2848 @@ -146,6 +146,8 @@ struct fsl_edma_slave_config {
2849         u32                             dev_addr;
2850         u32                             burst;
2851         u32                             attr;
2852 +       dma_addr_t                      dma_dev_addr;
2853 +       enum dma_data_direction         dma_dir;
2854  };
2855
2856  struct fsl_edma_chan {
2857 @@ -342,6 +344,53 @@ static int fsl_edma_resume(struct dma_ch
2858         return 0;
2859  }
2860
2861 +static void fsl_edma_unprep_slave_dma(struct fsl_edma_chan *fsl_chan)
2862 +{
2863 +       if (fsl_chan->fsc.dma_dir != DMA_NONE)
2864 +               dma_unmap_resource(fsl_chan->vchan.chan.device->dev,
2865 +                                  fsl_chan->fsc.dma_dev_addr,
2866 +                                  fsl_chan->fsc.burst, fsl_chan->fsc.dma_dir, 0);
2867 +       fsl_chan->fsc.dma_dir = DMA_NONE;
2868 +}
2869 +
2870 +static bool fsl_edma_prep_slave_dma(struct fsl_edma_chan *fsl_chan,
2871 +                                   enum dma_transfer_direction dir)
2872 +{
2873 +       struct device *dev = fsl_chan->vchan.chan.device->dev;
2874 +       enum dma_data_direction dma_dir;
2875 +
2876 +       switch (dir) {
2877 +       case DMA_MEM_TO_DEV:
2878 +               dma_dir = DMA_FROM_DEVICE;
2879 +               break;
2880 +       case DMA_DEV_TO_MEM:
2881 +               dma_dir = DMA_TO_DEVICE;
2882 +               break;
2883 +       case DMA_DEV_TO_DEV:
2884 +               dma_dir = DMA_BIDIRECTIONAL;
2885 +               break;
2886 +       default:
2887 +               dma_dir = DMA_NONE;
2888 +               break;
2889 +       }
2890 +
2891 +       /* Already mapped for this config? */
2892 +       if (fsl_chan->fsc.dma_dir == dma_dir)
2893 +               return true;
2894 +
2895 +       fsl_edma_unprep_slave_dma(fsl_chan);
2896 +       fsl_chan->fsc.dma_dev_addr = dma_map_resource(dev,
2897 +                                                     fsl_chan->fsc.dev_addr,
2898 +                                                     fsl_chan->fsc.burst,
2899 +                                                     dma_dir, 0);
2900 +       if (dma_mapping_error(dev, fsl_chan->fsc.dma_dev_addr))
2901 +               return false;
2902 +
2903 +       fsl_chan->fsc.dma_dir = dma_dir;
2904 +
2905 +       return true;
2906 +}
2907 +
2908  static int fsl_edma_slave_config(struct dma_chan *chan,
2909                                  struct dma_slave_config *cfg)
2910  {
2911 @@ -361,6 +410,7 @@ static int fsl_edma_slave_config(struct
2912         } else {
2913                         return -EINVAL;
2914         }
2915 +       fsl_edma_unprep_slave_dma(fsl_chan);
2916         return 0;
2917  }
2918
2919 @@ -553,6 +603,9 @@ static struct dma_async_tx_descriptor *f
2920         if (!is_slave_direction(fsl_chan->fsc.dir))
2921                 return NULL;
2922
2923 +       if (!fsl_edma_prep_slave_dma(fsl_chan, fsl_chan->fsc.dir))
2924 +               return NULL;
2925 +
2926         sg_len = buf_len / period_len;
2927         fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len);
2928         if (!fsl_desc)
2929 @@ -572,11 +625,11 @@ static struct dma_async_tx_descriptor *f
2930
2931                 if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) {
2932                         src_addr = dma_buf_next;
2933 -                       dst_addr = fsl_chan->fsc.dev_addr;
2934 +                       dst_addr = fsl_chan->fsc.dma_dev_addr;
2935                         soff = fsl_chan->fsc.addr_width;
2936                         doff = 0;
2937                 } else {
2938 -                       src_addr = fsl_chan->fsc.dev_addr;
2939 +                       src_addr = fsl_chan->fsc.dma_dev_addr;
2940                         dst_addr = dma_buf_next;
2941                         soff = 0;
2942                         doff = fsl_chan->fsc.addr_width;
2943 @@ -606,6 +659,9 @@ static struct dma_async_tx_descriptor *f
2944         if (!is_slave_direction(fsl_chan->fsc.dir))
2945                 return NULL;
2946
2947 +       if (!fsl_edma_prep_slave_dma(fsl_chan, fsl_chan->fsc.dir))
2948 +               return NULL;
2949 +
2950         fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len);
2951         if (!fsl_desc)
2952                 return NULL;
2953 @@ -618,11 +674,11 @@ static struct dma_async_tx_descriptor *f
2954
2955                 if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) {
2956                         src_addr = sg_dma_address(sg);
2957 -                       dst_addr = fsl_chan->fsc.dev_addr;
2958 +                       dst_addr = fsl_chan->fsc.dma_dev_addr;
2959                         soff = fsl_chan->fsc.addr_width;
2960                         doff = 0;
2961                 } else {
2962 -                       src_addr = fsl_chan->fsc.dev_addr;
2963 +                       src_addr = fsl_chan->fsc.dma_dev_addr;
2964                         dst_addr = sg_dma_address(sg);
2965                         soff = 0;
2966                         doff = fsl_chan->fsc.addr_width;
2967 @@ -802,6 +858,7 @@ static void fsl_edma_free_chan_resources
2968         fsl_edma_chan_mux(fsl_chan, 0, false);
2969         fsl_chan->edesc = NULL;
2970         vchan_get_all_descriptors(&fsl_chan->vchan, &head);
2971 +       fsl_edma_unprep_slave_dma(fsl_chan);
2972         spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);
2973
2974         vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
2975 @@ -937,6 +994,7 @@ static int fsl_edma_probe(struct platfor
2976                 fsl_chan->slave_id = 0;
2977                 fsl_chan->idle = true;
2978                 fsl_chan->vchan.desc_free = fsl_edma_free_desc;
2979 +               fsl_chan->fsc.dma_dir = DMA_NONE;
2980                 vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev);
2981
2982                 edma_writew(fsl_edma, 0x0, fsl_edma->membase + EDMA_TCD_CSR(i));
2983 --- /dev/null
2984 +++ b/drivers/dma/fsl-qdma.c
2985 @@ -0,0 +1,1278 @@
2986 +/*
2987 + * Driver for NXP Layerscape Queue direct memory access controller (qDMA)
2988 + *
2989 + * Copyright 2017 NXP
2990 + *
2991 + * Author:
2992 + *  Jiaheng Fan <jiaheng.fan@nxp.com>
2993 + *  Wen He <wen.he_1@nxp.com>
2994 + *
2995 + * SPDX-License-Identifier: GPL-2.0+
2996 + */
2997 +
2998 +#include <linux/interrupt.h>
2999 +#include <linux/module.h>
3000 +#include <linux/delay.h>
3001 +#include <linux/of_irq.h>
3002 +#include <linux/of_address.h>
3003 +#include <linux/of_platform.h>
3004 +#include <linux/of_dma.h>
3005 +#include <linux/dma-mapping.h>
3006 +#include <linux/dmapool.h>
3007 +#include <linux/dmaengine.h>
3008 +#include <linux/slab.h>
3009 +#include <linux/spinlock.h>
3010 +
3011 +#include "virt-dma.h"
3012 +
3013 +#define FSL_QDMA_DMR                   0x0
3014 +#define FSL_QDMA_DSR                   0x4
3015 +#define FSL_QDMA_DEIER                 0xe00
3016 +#define FSL_QDMA_DEDR                  0xe04
3017 +#define FSL_QDMA_DECFDW0R              0xe10
3018 +#define FSL_QDMA_DECFDW1R              0xe14
3019 +#define FSL_QDMA_DECFDW2R              0xe18
3020 +#define FSL_QDMA_DECFDW3R              0xe1c
3021 +#define FSL_QDMA_DECFQIDR              0xe30
3022 +#define FSL_QDMA_DECBR                 0xe34
3023 +
3024 +#define FSL_QDMA_BCQMR(x)              (0xc0 + 0x100 * (x))
3025 +#define FSL_QDMA_BCQSR(x)              (0xc4 + 0x100 * (x))
3026 +#define FSL_QDMA_BCQEDPA_SADDR(x)      (0xc8 + 0x100 * (x))
3027 +#define FSL_QDMA_BCQDPA_SADDR(x)       (0xcc + 0x100 * (x))
3028 +#define FSL_QDMA_BCQEEPA_SADDR(x)      (0xd0 + 0x100 * (x))
3029 +#define FSL_QDMA_BCQEPA_SADDR(x)       (0xd4 + 0x100 * (x))
3030 +#define FSL_QDMA_BCQIER(x)             (0xe0 + 0x100 * (x))
3031 +#define FSL_QDMA_BCQIDR(x)             (0xe4 + 0x100 * (x))
3032 +
3033 +#define FSL_QDMA_SQDPAR                        0x80c
3034 +#define FSL_QDMA_SQEPAR                        0x814
3035 +#define FSL_QDMA_BSQMR                 0x800
3036 +#define FSL_QDMA_BSQSR                 0x804
3037 +#define FSL_QDMA_BSQICR                        0x828
3038 +#define FSL_QDMA_CQMR                  0xa00
3039 +#define FSL_QDMA_CQDSCR1               0xa08
3040 +#define FSL_QDMA_CQDSCR2                0xa0c
3041 +#define FSL_QDMA_CQIER                 0xa10
3042 +#define FSL_QDMA_CQEDR                 0xa14
3043 +#define FSL_QDMA_SQCCMR                        0xa20
3044 +
3045 +#define FSL_QDMA_SQICR_ICEN
3046 +
3047 +#define FSL_QDMA_CQIDR_CQT             0xff000000
3048 +#define FSL_QDMA_CQIDR_SQPE            0x800000
3049 +#define FSL_QDMA_CQIDR_SQT             0x8000
3050 +
3051 +#define FSL_QDMA_BCQIER_CQTIE          0x8000
3052 +#define FSL_QDMA_BCQIER_CQPEIE         0x800000
3053 +#define FSL_QDMA_BSQICR_ICEN           0x80000000
3054 +#define FSL_QDMA_BSQICR_ICST(x)                ((x) << 16)
3055 +#define FSL_QDMA_CQIER_MEIE            0x80000000
3056 +#define FSL_QDMA_CQIER_TEIE            0x1
3057 +#define FSL_QDMA_SQCCMR_ENTER_WM       0x200000
3058 +
3059 +#define FSL_QDMA_QUEUE_MAX             8
3060 +
3061 +#define FSL_QDMA_BCQMR_EN              0x80000000
3062 +#define FSL_QDMA_BCQMR_EI              0x40000000
3063 +#define FSL_QDMA_BCQMR_CD_THLD(x)      ((x) << 20)
3064 +#define FSL_QDMA_BCQMR_CQ_SIZE(x)      ((x) << 16)
3065 +
3066 +#define FSL_QDMA_BCQSR_QF              0x10000
3067 +#define FSL_QDMA_BCQSR_XOFF            0x1
3068 +
3069 +#define FSL_QDMA_BSQMR_EN              0x80000000
3070 +#define FSL_QDMA_BSQMR_DI              0x40000000
3071 +#define FSL_QDMA_BSQMR_CQ_SIZE(x)      ((x) << 16)
3072 +
3073 +#define FSL_QDMA_BSQSR_QE              0x20000
3074 +
3075 +#define FSL_QDMA_DMR_DQD               0x40000000
3076 +#define FSL_QDMA_DSR_DB                        0x80000000
3077 +
3078 +#define FSL_QDMA_COMMAND_BUFFER_SIZE   64
3079 +#define FSL_QDMA_DESCRIPTOR_BUFFER_SIZE 32
3080 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN        64
3081 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX        16384
3082 +#define FSL_QDMA_QUEUE_NUM_MAX         8
3083 +
3084 +#define FSL_QDMA_CMD_RWTTYPE           0x4
3085 +#define FSL_QDMA_CMD_LWC                0x2
3086 +
3087 +#define FSL_QDMA_CMD_RWTTYPE_OFFSET    28
3088 +#define FSL_QDMA_CMD_NS_OFFSET         27
3089 +#define FSL_QDMA_CMD_DQOS_OFFSET       24
3090 +#define FSL_QDMA_CMD_WTHROTL_OFFSET    20
3091 +#define FSL_QDMA_CMD_DSEN_OFFSET       19
3092 +#define FSL_QDMA_CMD_LWC_OFFSET                16
3093 +
3094 +#define QDMA_CCDF_STATUS               20
3095 +#define QDMA_CCDF_OFFSET               20
3096 +#define QDMA_CCDF_MASK                 GENMASK(28, 20)
3097 +#define QDMA_CCDF_FOTMAT               BIT(29)
3098 +#define QDMA_CCDF_SER                  BIT(30)
3099 +
3100 +#define QDMA_SG_FIN                    BIT(30)
3101 +#define QDMA_SG_EXT                    BIT(31)
3102 +#define QDMA_SG_LEN_MASK               GENMASK(29, 0)
3103 +
3104 +#define QDMA_BIG_ENDIAN                        0x00000001
3105 +#define        COMP_TIMEOUT                    1000
3106 +#define COMMAND_QUEUE_OVERFLLOW                10
3107 +
3108 +#define QDMA_IN(fsl_qdma_engine, addr)                                 \
3109 +       (((fsl_qdma_engine)->big_endian & QDMA_BIG_ENDIAN) ?            \
3110 +               ioread32be(addr) : ioread32(addr))
3111 +#define QDMA_OUT(fsl_qdma_engine, addr, val)                           \
3112 +       (((fsl_qdma_engine)->big_endian & QDMA_BIG_ENDIAN) ?            \
3113 +               iowrite32be(val, addr) : iowrite32(val, addr))
3114 +
3115 +#define FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma_engine, x)                 \
3116 +       (((fsl_qdma_engine)->block_offset) * (x))
3117 +
3118 +static DEFINE_PER_CPU(u64, pre_addr);
3119 +static DEFINE_PER_CPU(u64, pre_queue);
3120 +
3121 +/* qDMA Command Descriptor Fotmats */
3122 +
3123 +struct fsl_qdma_format {
3124 +       __le32 status; /* ser, status */
3125 +       __le32 cfg;     /* format, offset */
3126 +       union {
3127 +               struct {
3128 +                       __le32 addr_lo; /* low 32-bits of 40-bit address */
3129 +                       u8 addr_hi;     /* high 8-bits of 40-bit address */
3130 +                       u8 __reserved1[2];
3131 +                       u8 cfg8b_w1; /* dd, queue */
3132 +               } __packed;
3133 +               __le64 data;
3134 +       };
3135 +} __packed;
3136 +
3137 +static inline u64
3138 +qdma_ccdf_addr_get64(const struct fsl_qdma_format *ccdf)
3139 +{
3140 +       return le64_to_cpu(ccdf->data) & 0xffffffffffLLU;
3141 +}
3142 +
3143 +static inline void
3144 +qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr)
3145 +{
3146 +       ccdf->addr_hi = upper_32_bits(addr);
3147 +       ccdf->addr_lo = cpu_to_le32(lower_32_bits(addr));
3148 +}
3149 +
3150 +static inline u64
3151 +qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf)
3152 +{
3153 +       return ccdf->cfg8b_w1 & 0xff;
3154 +}
3155 +
3156 +static inline int
3157 +qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf)
3158 +{
3159 +       return (le32_to_cpu(ccdf->cfg) & QDMA_CCDF_MASK) >> QDMA_CCDF_OFFSET;
3160 +}
3161 +
3162 +static inline void
3163 +qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset)
3164 +{
3165 +       ccdf->cfg = cpu_to_le32(QDMA_CCDF_FOTMAT | offset);
3166 +}
3167 +
3168 +static inline int
3169 +qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf)
3170 +{
3171 +       return (le32_to_cpu(ccdf->status) & QDMA_CCDF_MASK) >> QDMA_CCDF_STATUS;
3172 +}
3173 +
3174 +static inline void
3175 +qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status)
3176 +{
3177 +       ccdf->status = cpu_to_le32(QDMA_CCDF_SER | status);
3178 +}
3179 +
3180 +static inline void qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len)
3181 +{
3182 +       csgf->cfg = cpu_to_le32(len & QDMA_SG_LEN_MASK);
3183 +}
3184 +
3185 +static inline void qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len)
3186 +{
3187 +       csgf->cfg = cpu_to_le32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK));
3188 +}
3189 +
3190 +static inline void qdma_csgf_set_e(struct fsl_qdma_format *csgf, int len)
3191 +{
3192 +       csgf->cfg = cpu_to_le32(QDMA_SG_EXT | (len & QDMA_SG_LEN_MASK));
3193 +}
3194 +
3195 +/* qDMA Source Descriptor Format */
3196 +struct fsl_qdma_sdf {
3197 +       __le32 rev3;
3198 +       __le32 cfg; /* rev4, bit[0-11] - ssd, bit[12-23] sss */
3199 +       __le32 rev5;
3200 +       __le32 cmd;
3201 +} __packed;
3202 +
3203 +/* qDMA Destination Descriptor Format */
3204 +struct fsl_qdma_ddf {
3205 +       __le32 rev1;
3206 +       __le32 cfg; /* rev2, bit[0-11] - dsd, bit[12-23] - dss */
3207 +       __le32 rev3;
3208 +       __le32 cmd;
3209 +} __packed;
3210 +
3211 +struct fsl_qdma_chan {
3212 +       struct virt_dma_chan            vchan;
3213 +       struct virt_dma_desc            vdesc;
3214 +       enum dma_status                 status;
3215 +       struct fsl_qdma_engine          *qdma;
3216 +       struct fsl_qdma_queue           *queue;
3217 +};
3218 +
3219 +struct fsl_qdma_queue {
3220 +       struct fsl_qdma_format  *virt_head;
3221 +       struct fsl_qdma_format  *virt_tail;
3222 +       struct list_head        comp_used;
3223 +       struct list_head        comp_free;
3224 +       struct dma_pool         *comp_pool;
3225 +       struct dma_pool         *desc_pool;
3226 +       spinlock_t              queue_lock;
3227 +       dma_addr_t              bus_addr;
3228 +       u32                     n_cq;
3229 +       u32                     id;
3230 +       struct fsl_qdma_format  *cq;
3231 +       void __iomem            *block_base;
3232 +};
3233 +
3234 +struct fsl_qdma_comp {
3235 +       dma_addr_t              bus_addr;
3236 +       dma_addr_t              desc_bus_addr;
3237 +       void                    *virt_addr;
3238 +       void                    *desc_virt_addr;
3239 +       struct fsl_qdma_chan    *qchan;
3240 +       struct virt_dma_desc    vdesc;
3241 +       struct list_head        list;
3242 +};
3243 +
3244 +struct fsl_qdma_engine {
3245 +       struct dma_device       dma_dev;
3246 +       void __iomem            *ctrl_base;
3247 +       void __iomem            *status_base;
3248 +       void __iomem            *block_base;
3249 +       u32                     n_chans;
3250 +       u32                     n_queues;
3251 +       struct mutex            fsl_qdma_mutex;
3252 +       int                     error_irq;
3253 +       int                     *queue_irq;
3254 +       bool                    big_endian;
3255 +       struct fsl_qdma_queue   *queue;
3256 +       struct fsl_qdma_queue   **status;
3257 +       struct fsl_qdma_chan    *chans;
3258 +       int                     block_number;
3259 +       int                     block_offset;
3260 +       int                     irq_base;
3261 +       int                     desc_allocated;
3262 +
3263 +};
3264 +
3265 +static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr)
3266 +{
3267 +       return QDMA_IN(qdma, addr);
3268 +}
3269 +
3270 +static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val,
3271 +                                               void __iomem *addr)
3272 +{
3273 +       QDMA_OUT(qdma, addr, val);
3274 +}
3275 +
3276 +static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan)
3277 +{
3278 +       return container_of(chan, struct fsl_qdma_chan, vchan.chan);
3279 +}
3280 +
3281 +static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
3282 +{
3283 +       return container_of(vd, struct fsl_qdma_comp, vdesc);
3284 +}
3285 +
3286 +static void fsl_qdma_free_chan_resources(struct dma_chan *chan)
3287 +{
3288 +       struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3289 +       struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
3290 +       struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
3291 +       struct fsl_qdma_comp *comp_temp, *_comp_temp;
3292 +       unsigned long flags;
3293 +       LIST_HEAD(head);
3294 +
3295 +       spin_lock_irqsave(&fsl_chan->vchan.lock, flags);
3296 +       vchan_get_all_descriptors(&fsl_chan->vchan, &head);
3297 +       spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);
3298 +
3299 +       vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
3300 +
3301 +       if (!fsl_queue->comp_pool && !fsl_queue->comp_pool)
3302 +               return;
3303 +
3304 +       list_for_each_entry_safe(comp_temp, _comp_temp,
3305 +                               &fsl_queue->comp_used,  list) {
3306 +               dma_pool_free(fsl_queue->comp_pool,
3307 +                                        comp_temp->virt_addr,
3308 +                                        comp_temp->bus_addr);
3309 +               dma_pool_free(fsl_queue->desc_pool,
3310 +                               comp_temp->desc_virt_addr,
3311 +                               comp_temp->desc_bus_addr);
3312 +               list_del(&comp_temp->list);
3313 +               kfree(comp_temp);
3314 +       }
3315 +
3316 +       list_for_each_entry_safe(comp_temp, _comp_temp,
3317 +                               &fsl_queue->comp_free, list) {
3318 +               dma_pool_free(fsl_queue->comp_pool,
3319 +                                        comp_temp->virt_addr,
3320 +                                        comp_temp->bus_addr);
3321 +               dma_pool_free(fsl_queue->desc_pool,
3322 +                               comp_temp->desc_virt_addr,
3323 +                               comp_temp->desc_bus_addr);
3324 +               list_del(&comp_temp->list);
3325 +               kfree(comp_temp);
3326 +       }
3327 +
3328 +       dma_pool_destroy(fsl_queue->comp_pool);
3329 +       dma_pool_destroy(fsl_queue->desc_pool);
3330 +
3331 +       fsl_qdma->desc_allocated--;
3332 +       fsl_queue->comp_pool = NULL;
3333 +       fsl_queue->desc_pool = NULL;
3334 +}
3335 +
3336 +static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
3337 +                                       dma_addr_t dst, dma_addr_t src, u32 len)
3338 +{
3339 +       struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest;
3340 +       struct fsl_qdma_sdf *sdf;
3341 +       struct fsl_qdma_ddf *ddf;
3342 +
3343 +       ccdf = (struct fsl_qdma_format *)fsl_comp->virt_addr;
3344 +       csgf_desc = (struct fsl_qdma_format *)fsl_comp->virt_addr + 1;
3345 +       csgf_src = (struct fsl_qdma_format *)fsl_comp->virt_addr + 2;
3346 +       csgf_dest = (struct fsl_qdma_format *)fsl_comp->virt_addr + 3;
3347 +       sdf = (struct fsl_qdma_sdf *)fsl_comp->desc_virt_addr;
3348 +       ddf = (struct fsl_qdma_ddf *)fsl_comp->desc_virt_addr + 1;
3349 +
3350 +       memset(fsl_comp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE);
3351 +       memset(fsl_comp->desc_virt_addr, 0, FSL_QDMA_DESCRIPTOR_BUFFER_SIZE);
3352 +       /* Head Command Descriptor(Frame Descriptor) */
3353 +       qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16);
3354 +       qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf));
3355 +       qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf));
3356 +       /* Status notification is enqueued to status queue. */
3357 +       /* Compound Command Descriptor(Frame List Table) */
3358 +       qdma_desc_addr_set64(csgf_desc, fsl_comp->desc_bus_addr);
3359 +       /* It must be 32 as Compound S/G Descriptor */
3360 +       qdma_csgf_set_len(csgf_desc, 32);
3361 +       qdma_desc_addr_set64(csgf_src, src);
3362 +       qdma_csgf_set_len(csgf_src, len);
3363 +       qdma_desc_addr_set64(csgf_dest, dst);
3364 +       qdma_csgf_set_len(csgf_dest, len);
3365 +       /* This entry is the last entry. */
3366 +       qdma_csgf_set_f(csgf_dest, len);
3367 +       /* Descriptor Buffer */
3368 +       sdf->cmd = cpu_to_le32(
3369 +                       FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET);
3370 +       ddf->cmd = cpu_to_le32(
3371 +                       FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET);
3372 +       ddf->cmd |= cpu_to_le32(
3373 +                       FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET);
3374 +}
3375 +
3376 +/*
3377 + * Pre-request command descriptor and compound S/G for enqueue.
3378 + */
3379 +static int fsl_qdma_pre_request_enqueue_comp_desc(struct fsl_qdma_queue *queue)
3380 +{
3381 +       struct fsl_qdma_comp *comp_temp;
3382 +       int i;
3383 +
3384 +       for (i = 0; i < queue->n_cq + COMMAND_QUEUE_OVERFLLOW; i++) {
3385 +               comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
3386 +               if (!comp_temp)
3387 +                       return -ENOMEM;
3388 +               comp_temp->virt_addr = dma_pool_alloc(queue->comp_pool,
3389 +                                                     GFP_KERNEL,
3390 +                                                     &comp_temp->bus_addr);
3391 +
3392 +               if (!comp_temp->virt_addr) {
3393 +                       kfree(comp_temp);
3394 +                       return -ENOMEM;
3395 +               }
3396 +
3397 +               list_add_tail(&comp_temp->list, &queue->comp_free);
3398 +       }
3399 +
3400 +       return 0;
3401 +}
3402 +
3403 +/*
3404 + * Pre-request source and destination descriptor for enqueue.
3405 + */
3406 +static int fsl_qdma_pre_request_enqueue_sd_desc(struct fsl_qdma_queue *queue)
3407 +{
3408 +       struct fsl_qdma_comp *comp_temp, *_comp_temp;
3409 +
3410 +       list_for_each_entry_safe(comp_temp, _comp_temp,
3411 +                               &queue->comp_free, list) {
3412 +               comp_temp->desc_virt_addr = dma_pool_alloc(queue->desc_pool,
3413 +                                               GFP_KERNEL,
3414 +                                               &comp_temp->desc_bus_addr);
3415 +               if (!comp_temp->desc_virt_addr)
3416 +                       return -ENOMEM;
3417 +       }
3418 +
3419 +       return 0;
3420 +}
3421 +
3422 +/*
3423 + * Request a command descriptor for enqueue.
3424 + */
3425 +static struct fsl_qdma_comp *fsl_qdma_request_enqueue_desc(
3426 +                                       struct fsl_qdma_chan *fsl_chan)
3427 +{
3428 +       struct fsl_qdma_comp *comp_temp;
3429 +       struct fsl_qdma_queue *queue = fsl_chan->queue;
3430 +       unsigned long flags;
3431 +       int timeout = COMP_TIMEOUT;
3432 +
3433 +       while (timeout) {
3434 +               spin_lock_irqsave(&queue->queue_lock, flags);
3435 +               if (!list_empty(&queue->comp_free)) {
3436 +                       comp_temp = list_first_entry(&queue->comp_free,
3437 +                                            struct fsl_qdma_comp,
3438 +                                            list);
3439 +                       list_del(&comp_temp->list);
3440 +
3441 +                       spin_unlock_irqrestore(&queue->queue_lock, flags);
3442 +                       comp_temp->qchan = fsl_chan;
3443 +                       return comp_temp;
3444 +               }
3445 +               spin_unlock_irqrestore(&queue->queue_lock, flags);
3446 +               udelay(1);
3447 +               timeout--;
3448 +       }
3449 +
3450 +       return NULL;
3451 +}
3452 +
3453 +static struct fsl_qdma_queue *fsl_qdma_alloc_queue_resources(
3454 +                                       struct platform_device *pdev,
3455 +                                       struct fsl_qdma_engine *fsl_qdma)
3456 +{
3457 +       struct fsl_qdma_queue *queue_head, *queue_temp;
3458 +       int ret, len, i, j;
3459 +       unsigned int queue_size[FSL_QDMA_QUEUE_MAX];
3460 +       int queue_num;
3461 +       int block_number;
3462 +
3463 +       queue_num = fsl_qdma->n_queues;
3464 +       block_number = fsl_qdma->block_number;
3465 +
3466 +       if (queue_num > FSL_QDMA_QUEUE_MAX)
3467 +               queue_num = FSL_QDMA_QUEUE_MAX;
3468 +       len = sizeof(*queue_head) * queue_num * block_number;
3469 +       queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
3470 +       if (!queue_head)
3471 +               return NULL;
3472 +
3473 +       ret = device_property_read_u32_array(&pdev->dev, "queue-sizes",
3474 +                                       queue_size, queue_num);
3475 +       if (ret) {
3476 +               dev_err(&pdev->dev, "Can't get queue-sizes.\n");
3477 +               return NULL;
3478 +       }
3479 +       for (j = 0; j < block_number; j++) {
3480 +               for (i = 0; i < queue_num; i++) {
3481 +                       if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
3482 +                          queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
3483 +                               dev_err(&pdev->dev,
3484 +                                      "Get wrong queue-sizes.\n");
3485 +                               return NULL;
3486 +                       }
3487 +                       queue_temp = queue_head + i + (j * queue_num);
3488 +
3489 +                       queue_temp->cq =
3490 +                       dma_alloc_coherent(&pdev->dev,
3491 +                                       sizeof(struct fsl_qdma_format) *
3492 +                                       queue_size[i],
3493 +                                       &queue_temp->bus_addr,
3494 +                                       GFP_KERNEL);
3495 +                       if (!queue_temp->cq)
3496 +                               return NULL;
3497 +                       queue_temp->block_base = fsl_qdma->block_base +
3498 +                               FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
3499 +                       queue_temp->n_cq = queue_size[i];
3500 +                       queue_temp->id = i;
3501 +                       queue_temp->virt_head = queue_temp->cq;
3502 +                       queue_temp->virt_tail = queue_temp->cq;
3503 +                       /*
3504 +                        * List for queue command buffer
3505 +                        */
3506 +                       INIT_LIST_HEAD(&queue_temp->comp_used);
3507 +                       spin_lock_init(&queue_temp->queue_lock);
3508 +               }
3509 +       }
3510 +       return queue_head;
3511 +}
3512 +
3513 +static struct fsl_qdma_queue *fsl_qdma_prep_status_queue(
3514 +                                               struct platform_device *pdev)
3515 +{
3516 +       struct device_node *np = pdev->dev.of_node;
3517 +       struct fsl_qdma_queue *status_head;
3518 +       unsigned int status_size;
3519 +       int ret;
3520 +
3521 +       ret = of_property_read_u32(np, "status-sizes", &status_size);
3522 +       if (ret) {
3523 +               dev_err(&pdev->dev, "Can't get status-sizes.\n");
3524 +               return NULL;
3525 +       }
3526 +       if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX
3527 +                       || status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
3528 +               dev_err(&pdev->dev, "Get wrong status_size.\n");
3529 +               return NULL;
3530 +       }
3531 +       status_head = devm_kzalloc(&pdev->dev, sizeof(*status_head),
3532 +                                                               GFP_KERNEL);
3533 +       if (!status_head)
3534 +               return NULL;
3535 +
3536 +       /*
3537 +        * Buffer for queue command
3538 +        */
3539 +       status_head->cq = dma_alloc_coherent(&pdev->dev,
3540 +                                               sizeof(struct fsl_qdma_format) *
3541 +                                               status_size,
3542 +                                               &status_head->bus_addr,
3543 +                                               GFP_KERNEL);
3544 +       if (!status_head->cq)
3545 +               return NULL;
3546 +       status_head->n_cq = status_size;
3547 +       status_head->virt_head = status_head->cq;
3548 +       status_head->virt_tail = status_head->cq;
3549 +       status_head->comp_pool = NULL;
3550 +
3551 +       return status_head;
3552 +}
3553 +
3554 +static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)
3555 +{
3556 +       void __iomem *ctrl = fsl_qdma->ctrl_base;
3557 +       void __iomem *block;
3558 +       int i, count = 5;
3559 +       int j;
3560 +       u32 reg;
3561 +
3562 +       /* Disable the command queue and wait for idle state. */
3563 +       reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3564 +       reg |= FSL_QDMA_DMR_DQD;
3565 +       qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3566 +       for (j = 0; j < fsl_qdma->block_number; j++) {
3567 +               block = fsl_qdma->block_base +
3568 +                       FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
3569 +               for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++)
3570 +                       qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i));
3571 +       }
3572 +       while (1) {
3573 +               reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR);
3574 +               if (!(reg & FSL_QDMA_DSR_DB))
3575 +                       break;
3576 +               if (count-- < 0)
3577 +                       return -EBUSY;
3578 +               udelay(100);
3579 +       }
3580 +
3581 +       for (j = 0; j < fsl_qdma->block_number; j++) {
3582 +
3583 +               block = fsl_qdma->block_base +
3584 +                       FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
3585 +
3586 +               /* Disable status queue. */
3587 +               qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR);
3588 +
3589 +               /*
3590 +                * clear the command queue interrupt detect register for
3591 +                * all queues.
3592 +                */
3593 +               qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3594 +       }
3595 +
3596 +       return 0;
3597 +}
3598 +
3599 +static int fsl_qdma_queue_transfer_complete(
3600 +                               struct fsl_qdma_engine *fsl_qdma,
3601 +                               void *block,
3602 +                               int id)
3603 +{
3604 +       struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
3605 +       struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id];
3606 +       struct fsl_qdma_queue *temp_queue;
3607 +       struct fsl_qdma_format *status_addr;
3608 +       struct fsl_qdma_comp *fsl_comp = NULL;
3609 +       u32 reg, i;
3610 +       bool duplicate, duplicate_handle;
3611 +
3612 +       while (1) {
3613 +               duplicate = 0;
3614 +               duplicate_handle = 0;
3615 +               reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR);
3616 +               if (reg & FSL_QDMA_BSQSR_QE)
3617 +                       return 0;
3618 +
3619 +               status_addr = fsl_status->virt_head;
3620 +
3621 +               if (qdma_ccdf_get_queue(status_addr) ==
3622 +                  __this_cpu_read(pre_queue) &&
3623 +                       qdma_ccdf_addr_get64(status_addr) ==
3624 +                       __this_cpu_read(pre_addr))
3625 +                       duplicate = 1;
3626 +               i = qdma_ccdf_get_queue(status_addr) +
3627 +                       id * fsl_qdma->n_queues;
3628 +               __this_cpu_write(pre_addr, qdma_ccdf_addr_get64(status_addr));
3629 +               __this_cpu_write(pre_queue, qdma_ccdf_get_queue(status_addr));
3630 +               temp_queue = fsl_queue + i;
3631 +
3632 +               spin_lock(&temp_queue->queue_lock);
3633 +               if (list_empty(&temp_queue->comp_used)) {
3634 +                       if (duplicate)
3635 +                               duplicate_handle = 1;
3636 +                       else {
3637 +                               spin_unlock(&temp_queue->queue_lock);
3638 +                               return -1;
3639 +                       }
3640 +               } else {
3641 +                       fsl_comp = list_first_entry(&temp_queue->comp_used,
3642 +                                                       struct fsl_qdma_comp,
3643 +                                                       list);
3644 +                       if (fsl_comp->bus_addr + 16 !=
3645 +                               __this_cpu_read(pre_addr)) {
3646 +                               if (duplicate)
3647 +                                       duplicate_handle = 1;
3648 +                               else {
3649 +                                       spin_unlock(&temp_queue->queue_lock);
3650 +                                       return -1;
3651 +                               }
3652 +                       }
3653 +
3654 +               }
3655 +
3656 +               if (duplicate_handle) {
3657 +                       reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
3658 +                       reg |= FSL_QDMA_BSQMR_DI;
3659 +                       qdma_desc_addr_set64(status_addr, 0x0);
3660 +                       fsl_status->virt_head++;
3661 +                       if (fsl_status->virt_head == fsl_status->cq
3662 +                                                  + fsl_status->n_cq)
3663 +                               fsl_status->virt_head = fsl_status->cq;
3664 +                       qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3665 +                       spin_unlock(&temp_queue->queue_lock);
3666 +                       continue;
3667 +               }
3668 +               list_del(&fsl_comp->list);
3669 +
3670 +               reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
3671 +               reg |= FSL_QDMA_BSQMR_DI;
3672 +               qdma_desc_addr_set64(status_addr, 0x0);
3673 +               fsl_status->virt_head++;
3674 +               if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq)
3675 +                       fsl_status->virt_head = fsl_status->cq;
3676 +               qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3677 +               spin_unlock(&temp_queue->queue_lock);
3678 +
3679 +               spin_lock(&fsl_comp->qchan->vchan.lock);
3680 +               vchan_cookie_complete(&fsl_comp->vdesc);
3681 +               fsl_comp->qchan->status = DMA_COMPLETE;
3682 +               spin_unlock(&fsl_comp->qchan->vchan.lock);
3683 +       }
3684 +       return 0;
3685 +}
3686 +
3687 +static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id)
3688 +{
3689 +       struct fsl_qdma_engine *fsl_qdma = dev_id;
3690 +       unsigned int intr;
3691 +       void __iomem *status = fsl_qdma->status_base;
3692 +
3693 +       intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR);
3694 +
3695 +       if (intr)
3696 +               dev_err(fsl_qdma->dma_dev.dev, "DMA transaction error!\n");
3697 +
3698 +       qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR);
3699 +       return IRQ_HANDLED;
3700 +}
3701 +
3702 +static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id)
3703 +{
3704 +       struct fsl_qdma_engine *fsl_qdma = dev_id;
3705 +       unsigned int intr, reg;
3706 +       void __iomem *ctrl = fsl_qdma->ctrl_base;
3707 +       void __iomem *block;
3708 +       int id;
3709 +
3710 +       id = irq - fsl_qdma->irq_base;
3711 +       if (id < 0 && id > fsl_qdma->block_number) {
3712 +               dev_err(fsl_qdma->dma_dev.dev,
3713 +                       "irq %d is wrong irq_base is %d\n",
3714 +                       irq, fsl_qdma->irq_base);
3715 +       }
3716 +
3717 +       block = fsl_qdma->block_base +
3718 +               FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id);
3719 +
3720 +       intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0));
3721 +
3722 +       if ((intr & FSL_QDMA_CQIDR_SQT) != 0)
3723 +               intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id);
3724 +
3725 +       if (intr != 0) {
3726 +               reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3727 +               reg |= FSL_QDMA_DMR_DQD;
3728 +               qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3729 +               qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0));
3730 +               dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n");
3731 +       }
3732 +
3733 +       qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3734 +
3735 +       return IRQ_HANDLED;
3736 +}
3737 +
3738 +static int
3739 +fsl_qdma_irq_init(struct platform_device *pdev,
3740 +                 struct fsl_qdma_engine *fsl_qdma)
3741 +{
3742 +       char irq_name[20];
3743 +       int i;
3744 +       int cpu;
3745 +       int ret;
3746 +
3747 +       fsl_qdma->error_irq = platform_get_irq_byname(pdev,
3748 +                                                       "qdma-error");
3749 +       if (fsl_qdma->error_irq < 0) {
3750 +               dev_err(&pdev->dev, "Can't get qdma controller irq.\n");
3751 +               return fsl_qdma->error_irq;
3752 +       }
3753 +
3754 +       ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq,
3755 +                       fsl_qdma_error_handler, 0, "qDMA error", fsl_qdma);
3756 +       if (ret) {
3757 +               dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n");
3758 +               return  ret;
3759 +       }
3760 +
3761 +       for (i = 0; i < fsl_qdma->block_number; i++) {
3762 +               sprintf(irq_name, "qdma-queue%d", i);
3763 +               fsl_qdma->queue_irq[i] = platform_get_irq_byname(pdev,
3764 +                                                               irq_name);
3765 +
3766 +               if (fsl_qdma->queue_irq[i] < 0) {
3767 +                       dev_err(&pdev->dev,
3768 +                                  "Can't get qdma queue %d irq.\n",
3769 +                                  i);
3770 +                       return fsl_qdma->queue_irq[i];
3771 +               }
3772 +
3773 +               ret = devm_request_irq(&pdev->dev,
3774 +                                     fsl_qdma->queue_irq[i],
3775 +                                     fsl_qdma_queue_handler,
3776 +                                     0,
3777 +                                     "qDMA queue",
3778 +                                     fsl_qdma);
3779 +               if (ret) {
3780 +                       dev_err(&pdev->dev,
3781 +                              "Can't register qDMA queue IRQ.\n");
3782 +                       return  ret;
3783 +               }
3784 +
3785 +               cpu = i % num_online_cpus();
3786 +               ret = irq_set_affinity_hint(fsl_qdma->queue_irq[i],
3787 +                                          get_cpu_mask(cpu));
3788 +               if (ret) {
3789 +                       dev_err(&pdev->dev,
3790 +                              "Can't set cpu %d affinity to IRQ %d.\n",
3791 +                               cpu,
3792 +                               fsl_qdma->queue_irq[i]);
3793 +                       return  ret;
3794 +               }
3795 +
3796 +       }
3797 +
3798 +       return 0;
3799 +}
3800 +
3801 +static void fsl_qdma_irq_exit(
3802 +               struct platform_device *pdev, struct fsl_qdma_engine *fsl_qdma)
3803 +{
3804 +       if (fsl_qdma->queue_irq[0] == fsl_qdma->error_irq) {
3805 +               devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[0], fsl_qdma);
3806 +       } else {
3807 +               devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[0], fsl_qdma);
3808 +               devm_free_irq(&pdev->dev, fsl_qdma->error_irq, fsl_qdma);
3809 +       }
3810 +}
3811 +
3812 +static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
3813 +{
3814 +       struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
3815 +       struct fsl_qdma_queue *temp;
3816 +       void __iomem *ctrl = fsl_qdma->ctrl_base;
3817 +       void __iomem *status = fsl_qdma->status_base;
3818 +       void __iomem *block;
3819 +       int i, j, ret;
3820 +       u32 reg;
3821 +
3822 +       /* Try to halt the qDMA engine first. */
3823 +       ret = fsl_qdma_halt(fsl_qdma);
3824 +       if (ret) {
3825 +               dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!");
3826 +               return ret;
3827 +       }
3828 +
3829 +       for (i = 0; i < fsl_qdma->block_number; i++) {
3830 +               /*
3831 +                * Clear the command queue interrupt detect register for
3832 +                * all queues.
3833 +                */
3834 +
3835 +               block = fsl_qdma->block_base +
3836 +                       FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, i);
3837 +               qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3838 +       }
3839 +
3840 +       for (j = 0; j < fsl_qdma->block_number; j++) {
3841 +               block = fsl_qdma->block_base +
3842 +                       FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
3843 +               for (i = 0; i < fsl_qdma->n_queues; i++) {
3844 +                       temp = fsl_queue + i + (j * fsl_qdma->n_queues);
3845 +                       /*
3846 +                        * Initialize Command Queue registers to
3847 +                        * point to the first
3848 +                        * command descriptor in memory.
3849 +                        * Dequeue Pointer Address Registers
3850 +                        * Enqueue Pointer Address Registers
3851 +                        */
3852 +
3853 +                       qdma_writel(fsl_qdma, temp->bus_addr,
3854 +                                          block + FSL_QDMA_BCQDPA_SADDR(i));
3855 +                       qdma_writel(fsl_qdma, temp->bus_addr,
3856 +                                          block + FSL_QDMA_BCQEPA_SADDR(i));
3857 +
3858 +                       /* Initialize the queue mode. */
3859 +                       reg = FSL_QDMA_BCQMR_EN;
3860 +                       reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4);
3861 +                       reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6);
3862 +                       qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i));
3863 +               }
3864 +
3865 +               /*
3866 +                * Workaround for erratum: ERR010812.
3867 +                * We must enable XOFF to avoid the enqueue rejection occurs.
3868 +                * Setting SQCCMR ENTER_WM to 0x20.
3869 +                */
3870 +
3871 +               qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM,
3872 +                                  block + FSL_QDMA_SQCCMR);
3873 +
3874 +               /*
3875 +                * Initialize status queue registers to point to the first
3876 +                * command descriptor in memory.
3877 +                * Dequeue Pointer Address Registers
3878 +                * Enqueue Pointer Address Registers
3879 +                */
3880 +
3881 +               qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr,
3882 +                                  block + FSL_QDMA_SQEPAR);
3883 +               qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr,
3884 +                                  block + FSL_QDMA_SQDPAR);
3885 +               /* Initialize status queue interrupt. */
3886 +               qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE,
3887 +                                  block + FSL_QDMA_BCQIER(0));
3888 +               qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN |
3889 +                                  FSL_QDMA_BSQICR_ICST(5) | 0x8000,
3890 +                                  block + FSL_QDMA_BSQICR);
3891 +               qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE |
3892 +                                  FSL_QDMA_CQIER_TEIE,
3893 +                                  block + FSL_QDMA_CQIER);
3894 +
3895 +               /* Initialize the status queue mode. */
3896 +               reg = FSL_QDMA_BSQMR_EN;
3897 +               reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2(
3898 +                       fsl_qdma->status[j]->n_cq) - 6);
3899 +
3900 +               qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3901 +               reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
3902 +
3903 +       }
3904 +
3905 +       /* Initialize controller interrupt register. */
3906 +       qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR);
3907 +       qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEIER);
3908 +
3909 +       reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3910 +       reg &= ~FSL_QDMA_DMR_DQD;
3911 +       qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3912 +
3913 +       return 0;
3914 +}
3915 +
3916 +static struct dma_async_tx_descriptor *
3917 +fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
3918 +               dma_addr_t src, size_t len, unsigned long flags)
3919 +{
3920 +       struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3921 +       struct fsl_qdma_comp *fsl_comp;
3922 +
3923 +       fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan);
3924 +
3925 +       if (!fsl_comp)
3926 +               return NULL;
3927 +
3928 +       fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len);
3929 +
3930 +       return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags);
3931 +}
3932 +
3933 +static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan)
3934 +{
3935 +       struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
3936 +       struct fsl_qdma_comp *fsl_comp;
3937 +       struct virt_dma_desc *vdesc;
3938 +       void __iomem *block = fsl_queue->block_base;
3939 +       u32 reg;
3940 +
3941 +       reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id));
3942 +       if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF))
3943 +               return;
3944 +       vdesc = vchan_next_desc(&fsl_chan->vchan);
3945 +       if (!vdesc)
3946 +               return;
3947 +       list_del(&vdesc->node);
3948 +       fsl_comp = to_fsl_qdma_comp(vdesc);
3949 +
3950 +       memcpy(fsl_queue->virt_head++, fsl_comp->virt_addr, 16);
3951 +       if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq)
3952 +               fsl_queue->virt_head = fsl_queue->cq;
3953 +
3954 +       list_add_tail(&fsl_comp->list, &fsl_queue->comp_used);
3955 +       barrier();
3956 +       reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id));
3957 +       reg |= FSL_QDMA_BCQMR_EI;
3958 +       qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
3959 +       fsl_chan->status = DMA_IN_PROGRESS;
3960 +}
3961 +
3962 +static enum dma_status fsl_qdma_tx_status(struct dma_chan *chan,
3963 +               dma_cookie_t cookie, struct dma_tx_state *txstate)
3964 +{
3965 +       return dma_cookie_status(chan, cookie, txstate);
3966 +}
3967 +
3968 +static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc)
3969 +{
3970 +       struct fsl_qdma_comp *fsl_comp;
3971 +       struct fsl_qdma_queue *fsl_queue;
3972 +       unsigned long flags;
3973 +
3974 +       fsl_comp = to_fsl_qdma_comp(vdesc);
3975 +       fsl_queue = fsl_comp->qchan->queue;
3976 +
3977 +       spin_lock_irqsave(&fsl_queue->queue_lock, flags);
3978 +       list_add_tail(&fsl_comp->list, &fsl_queue->comp_free);
3979 +       spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
3980 +}
3981 +
3982 +static void fsl_qdma_issue_pending(struct dma_chan *chan)
3983 +{
3984 +       struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3985 +       struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
3986 +       unsigned long flags;
3987 +
3988 +       spin_lock_irqsave(&fsl_queue->queue_lock, flags);
3989 +       spin_lock(&fsl_chan->vchan.lock);
3990 +       if (vchan_issue_pending(&fsl_chan->vchan))
3991 +               fsl_qdma_enqueue_desc(fsl_chan);
3992 +       spin_unlock(&fsl_chan->vchan.lock);
3993 +       spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
3994 +}
3995 +
3996 +static void fsl_qdma_synchronize(struct dma_chan *chan)
3997 +{
3998 +       struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3999 +
4000 +       vchan_synchronize(&fsl_chan->vchan);
4001 +}
4002 +
4003 +static int fsl_qdma_terminate_all(struct dma_chan *chan)
4004 +{
4005 +       struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
4006 +       unsigned long flags;
4007 +       LIST_HEAD(head);
4008 +
4009 +       spin_lock_irqsave(&fsl_chan->vchan.lock, flags);
4010 +       vchan_get_all_descriptors(&fsl_chan->vchan, &head);
4011 +       spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);
4012 +       vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
4013 +       return 0;
4014 +}
4015 +
4016 +static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan)
4017 +{
4018 +       struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
4019 +       struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
4020 +       struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
4021 +       int ret;
4022 +
4023 +       if (fsl_queue->comp_pool && fsl_queue->desc_pool)
4024 +               return fsl_qdma->desc_allocated;
4025 +
4026 +       INIT_LIST_HEAD(&fsl_queue->comp_free);
4027 +
4028 +       /*
4029 +        * The dma pool for queue command buffer
4030 +        */
4031 +       fsl_queue->comp_pool =
4032 +       dma_pool_create("comp_pool",
4033 +                      chan->device->dev,
4034 +                      FSL_QDMA_COMMAND_BUFFER_SIZE,
4035 +                      64, 0);
4036 +       if (!fsl_queue->comp_pool)
4037 +               return -ENOMEM;
4038 +
4039 +       /*
4040 +        * The dma pool for Descriptor(SD/DD) buffer
4041 +        */
4042 +       fsl_queue->desc_pool =
4043 +       dma_pool_create("desc_pool",
4044 +                      chan->device->dev,
4045 +                      FSL_QDMA_DESCRIPTOR_BUFFER_SIZE,
4046 +                      32, 0);
4047 +       if (!fsl_queue->desc_pool)
4048 +               goto err_desc_pool;
4049 +
4050 +       ret = fsl_qdma_pre_request_enqueue_comp_desc(fsl_queue);
4051 +       if (ret) {
4052 +               dev_err(chan->device->dev, "failed to alloc dma buffer for "
4053 +                               "comp S/G descriptor\n");
4054 +               goto err_mem;
4055 +       }
4056 +
4057 +       ret = fsl_qdma_pre_request_enqueue_sd_desc(fsl_queue);
4058 +       if (ret) {
4059 +               dev_err(chan->device->dev, "failed to alloc dma buffer for "
4060 +                               "S/D descriptor\n");
4061 +               goto err_mem;
4062 +       }
4063 +
4064 +       fsl_qdma->desc_allocated++;
4065 +       return fsl_qdma->desc_allocated;
4066 +
4067 +err_mem:
4068 +       dma_pool_destroy(fsl_queue->desc_pool);
4069 +err_desc_pool:
4070 +       dma_pool_destroy(fsl_queue->comp_pool);
4071 +       return -ENOMEM;
4072 +}
4073 +
4074 +static int fsl_qdma_probe(struct platform_device *pdev)
4075 +{
4076 +       struct device_node *np = pdev->dev.of_node;
4077 +       struct fsl_qdma_engine *fsl_qdma;
4078 +       struct fsl_qdma_chan *fsl_chan;
4079 +       struct resource *res;
4080 +       unsigned int len, chans, queues;
4081 +       int ret, i;
4082 +       int blk_num;
4083 +       int blk_off;
4084 +
4085 +       ret = of_property_read_u32(np, "channels", &chans);
4086 +       if (ret) {
4087 +               dev_err(&pdev->dev, "Can't get channels.\n");
4088 +               return ret;
4089 +       }
4090 +
4091 +       ret = of_property_read_u32(np, "block-offset", &blk_off);
4092 +       if (ret) {
4093 +               dev_err(&pdev->dev, "Can't get block-offset.\n");
4094 +               return ret;
4095 +       }
4096 +
4097 +       ret = of_property_read_u32(np, "block-number", &blk_num);
4098 +       if (ret) {
4099 +               dev_err(&pdev->dev, "Can't get block-number.\n");
4100 +               return ret;
4101 +       }
4102 +
4103 +       blk_num = min_t(int, blk_num, num_online_cpus());
4104 +
4105 +       len = sizeof(*fsl_qdma);
4106 +       fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
4107 +       if (!fsl_qdma)
4108 +               return -ENOMEM;
4109 +
4110 +       len = sizeof(*fsl_chan) * chans;
4111 +       fsl_qdma->chans = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
4112 +       if (!fsl_qdma->chans)
4113 +               return -ENOMEM;
4114 +
4115 +       len = sizeof(struct fsl_qdma_queue *) * blk_num;
4116 +       fsl_qdma->status = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
4117 +       if (!fsl_qdma->status)
4118 +               return -ENOMEM;
4119 +
4120 +       len = sizeof(int) * blk_num;
4121 +       fsl_qdma->queue_irq = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
4122 +       if (!fsl_qdma->queue_irq)
4123 +               return -ENOMEM;
4124 +
4125 +       ret = of_property_read_u32(np, "queues", &queues);
4126 +       if (ret) {
4127 +               dev_err(&pdev->dev, "Can't get queues.\n");
4128 +               return ret;
4129 +       }
4130 +
4131 +       fsl_qdma->desc_allocated = 0;
4132 +       fsl_qdma->n_chans = chans;
4133 +       fsl_qdma->n_queues = queues;
4134 +       fsl_qdma->block_number = blk_num;
4135 +       fsl_qdma->block_offset = blk_off;
4136 +
4137 +       mutex_init(&fsl_qdma->fsl_qdma_mutex);
4138 +
4139 +       for (i = 0; i < fsl_qdma->block_number; i++) {
4140 +               fsl_qdma->status[i] = fsl_qdma_prep_status_queue(pdev);
4141 +               if (!fsl_qdma->status[i])
4142 +                       return -ENOMEM;
4143 +       }
4144 +       res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4145 +       fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res);
4146 +       if (IS_ERR(fsl_qdma->ctrl_base))
4147 +               return PTR_ERR(fsl_qdma->ctrl_base);
4148 +
4149 +       res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
4150 +       fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res);
4151 +       if (IS_ERR(fsl_qdma->status_base))
4152 +               return PTR_ERR(fsl_qdma->status_base);
4153 +
4154 +       res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
4155 +       fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res);
4156 +       if (IS_ERR(fsl_qdma->block_base))
4157 +               return PTR_ERR(fsl_qdma->block_base);
4158 +       fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, fsl_qdma);
4159 +       if (!fsl_qdma->queue)
4160 +               return -ENOMEM;
4161 +
4162 +       ret = fsl_qdma_irq_init(pdev, fsl_qdma);
4163 +       if (ret)
4164 +               return ret;
4165 +
4166 +       fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0");
4167 +       fsl_qdma->big_endian = of_property_read_bool(np, "big-endian");
4168 +       INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels);
4169 +
4170 +       for (i = 0; i < fsl_qdma->n_chans; i++) {
4171 +               struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
4172 +
4173 +               fsl_chan->qdma = fsl_qdma;
4174 +               fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues *
4175 +                                                       fsl_qdma->block_number);
4176 +               fsl_chan->vchan.desc_free = fsl_qdma_free_desc;
4177 +               vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev);
4178 +       }
4179 +
4180 +       dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask);
4181 +
4182 +       fsl_qdma->dma_dev.dev = &pdev->dev;
4183 +       fsl_qdma->dma_dev.device_free_chan_resources
4184 +               = fsl_qdma_free_chan_resources;
4185 +       fsl_qdma->dma_dev.device_alloc_chan_resources
4186 +               = fsl_qdma_alloc_chan_resources;
4187 +       fsl_qdma->dma_dev.device_tx_status = fsl_qdma_tx_status;
4188 +       fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy;
4189 +       fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending;
4190 +       fsl_qdma->dma_dev.device_synchronize = fsl_qdma_synchronize;
4191 +       fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all;
4192 +
4193 +       dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
4194 +
4195 +       platform_set_drvdata(pdev, fsl_qdma);
4196 +
4197 +       ret = dma_async_device_register(&fsl_qdma->dma_dev);
4198 +       if (ret) {
4199 +               dev_err(&pdev->dev,
4200 +                       "Can't register NXP Layerscape qDMA engine.\n");
4201 +               return ret;
4202 +       }
4203 +
4204 +       ret = fsl_qdma_reg_init(fsl_qdma);
4205 +       if (ret) {
4206 +               dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n");
4207 +               return ret;
4208 +       }
4209 +
4210 +       return 0;
4211 +}
4212 +
4213 +static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev)
4214 +{
4215 +       struct fsl_qdma_chan *chan, *_chan;
4216 +
4217 +       list_for_each_entry_safe(chan, _chan,
4218 +                               &dmadev->channels, vchan.chan.device_node) {
4219 +               list_del(&chan->vchan.chan.device_node);
4220 +               tasklet_kill(&chan->vchan.task);
4221 +       }
4222 +}
4223 +
4224 +static int fsl_qdma_remove(struct platform_device *pdev)
4225 +{
4226 +       struct device_node *np = pdev->dev.of_node;
4227 +       struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev);
4228 +       struct fsl_qdma_queue *status;
4229 +       int i;
4230 +
4231 +       fsl_qdma_irq_exit(pdev, fsl_qdma);
4232 +       fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev);
4233 +       of_dma_controller_free(np);
4234 +       dma_async_device_unregister(&fsl_qdma->dma_dev);
4235 +
4236 +       for (i = 0; i < fsl_qdma->block_number; i++) {
4237 +               status = fsl_qdma->status[i];
4238 +               dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_format) *
4239 +                               status->n_cq, status->cq, status->bus_addr);
4240 +       }
4241 +       return 0;
4242 +}
4243 +
4244 +static const struct of_device_id fsl_qdma_dt_ids[] = {
4245 +       { .compatible = "fsl,ls1021a-qdma", },
4246 +       { /* sentinel */ }
4247 +};
4248 +MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids);
4249 +
4250 +static struct platform_driver fsl_qdma_driver = {
4251 +       .driver         = {
4252 +               .name   = "fsl-qdma",
4253 +               .of_match_table = fsl_qdma_dt_ids,
4254 +       },
4255 +       .probe          = fsl_qdma_probe,
4256 +       .remove         = fsl_qdma_remove,
4257 +};
4258 +
4259 +module_platform_driver(fsl_qdma_driver);
4260 +
4261 +MODULE_ALIAS("platform:fsl-qdma");
4262 +MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver");
4263 +MODULE_LICENSE("GPL v2");