drivers/dma/owl-dma.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 //
   3 // Actions Semi Owl SoCs DMA driver
   4 //
   5 // Copyright (c) 2014 Actions Semi Inc.
   6 // Author: David Liu <liuwei@actions-semi.com>
   7 //
   8 // Copyright (c) 2018 Linaro Ltd.
   9 // Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
  10
  11 #include <linux/bitops.h>
  12 #include <linux/clk.h>
  13 #include <linux/delay.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/dmapool.h>
  17 #include <linux/err.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/io.h>
  21 #include <linux/mm.h>
  22 #include <linux/module.h>
  23 #include <linux/of_device.h>
  24 #include <linux/of_dma.h>
  25 #include <linux/slab.h>
  26 #include "virt-dma.h"
  27
  28 #define OWL_DMA_FRAME_MAX_LENGTH                0xfffff
  29
  30 /* Global DMA Controller Registers */
  31 #define OWL_DMA_IRQ_PD0                         0x00
  32 #define OWL_DMA_IRQ_PD1                         0x04
  33 #define OWL_DMA_IRQ_PD2                         0x08
  34 #define OWL_DMA_IRQ_PD3                         0x0C
  35 #define OWL_DMA_IRQ_EN0                         0x10
  36 #define OWL_DMA_IRQ_EN1                         0x14
  37 #define OWL_DMA_IRQ_EN2                         0x18
  38 #define OWL_DMA_IRQ_EN3                         0x1C
  39 #define OWL_DMA_SECURE_ACCESS_CTL               0x20
  40 #define OWL_DMA_NIC_QOS                         0x24
  41 #define OWL_DMA_DBGSEL                          0x28
  42 #define OWL_DMA_IDLE_STAT                       0x2C
  43
  44 /* Channel Registers */
  45 #define OWL_DMA_CHAN_BASE(i)                    (0x100 + (i) * 0x100)
  46 #define OWL_DMAX_MODE                           0x00
  47 #define OWL_DMAX_SOURCE                         0x04
  48 #define OWL_DMAX_DESTINATION                    0x08
  49 #define OWL_DMAX_FRAME_LEN                      0x0C
  50 #define OWL_DMAX_FRAME_CNT                      0x10
  51 #define OWL_DMAX_REMAIN_FRAME_CNT               0x14
  52 #define OWL_DMAX_REMAIN_CNT                     0x18
  53 #define OWL_DMAX_SOURCE_STRIDE                  0x1C
  54 #define OWL_DMAX_DESTINATION_STRIDE             0x20
  55 #define OWL_DMAX_START                          0x24
  56 #define OWL_DMAX_PAUSE                          0x28
  57 #define OWL_DMAX_CHAINED_CTL                    0x2C
  58 #define OWL_DMAX_CONSTANT                       0x30
  59 #define OWL_DMAX_LINKLIST_CTL                   0x34
  60 #define OWL_DMAX_NEXT_DESCRIPTOR                0x38
  61 #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM         0x3C
  62 #define OWL_DMAX_INT_CTL                        0x40
  63 #define OWL_DMAX_INT_STATUS                     0x44
  64 #define OWL_DMAX_CURRENT_SOURCE_POINTER         0x48
  65 #define OWL_DMAX_CURRENT_DESTINATION_POINTER    0x4C
  66
  67 /* OWL_DMAX_MODE Bits */
  68 #define OWL_DMA_MODE_TS(x)                      (((x) & GENMASK(5, 0)) << 0)
  69 #define OWL_DMA_MODE_ST(x)                      (((x) & GENMASK(1, 0)) << 8)
  70 #define OWL_DMA_MODE_ST_DEV                     OWL_DMA_MODE_ST(0)
  71 #define OWL_DMA_MODE_ST_DCU                     OWL_DMA_MODE_ST(2)
  72 #define OWL_DMA_MODE_ST_SRAM                    OWL_DMA_MODE_ST(3)
  73 #define OWL_DMA_MODE_DT(x)                      (((x) & GENMASK(1, 0)) << 10)
  74 #define OWL_DMA_MODE_DT_DEV                     OWL_DMA_MODE_DT(0)
  75 #define OWL_DMA_MODE_DT_DCU                     OWL_DMA_MODE_DT(2)
  76 #define OWL_DMA_MODE_DT_SRAM                    OWL_DMA_MODE_DT(3)
  77 #define OWL_DMA_MODE_SAM(x)                     (((x) & GENMASK(1, 0)) << 16)
  78 #define OWL_DMA_MODE_SAM_CONST                  OWL_DMA_MODE_SAM(0)
  79 #define OWL_DMA_MODE_SAM_INC                    OWL_DMA_MODE_SAM(1)
  80 #define OWL_DMA_MODE_SAM_STRIDE                 OWL_DMA_MODE_SAM(2)
  81 #define OWL_DMA_MODE_DAM(x)                     (((x) & GENMASK(1, 0)) << 18)
  82 #define OWL_DMA_MODE_DAM_CONST                  OWL_DMA_MODE_DAM(0)
  83 #define OWL_DMA_MODE_DAM_INC                    OWL_DMA_MODE_DAM(1)
  84 #define OWL_DMA_MODE_DAM_STRIDE                 OWL_DMA_MODE_DAM(2)
  85 #define OWL_DMA_MODE_PW(x)                      (((x) & GENMASK(2, 0)) << 20)
  86 #define OWL_DMA_MODE_CB                         BIT(23)
  87 #define OWL_DMA_MODE_NDDBW(x)                   (((x) & 0x1) << 28)
  88 #define OWL_DMA_MODE_NDDBW_32BIT                OWL_DMA_MODE_NDDBW(0)
  89 #define OWL_DMA_MODE_NDDBW_8BIT                 OWL_DMA_MODE_NDDBW(1)
  90 #define OWL_DMA_MODE_CFE                        BIT(29)
  91 #define OWL_DMA_MODE_LME                        BIT(30)
  92 #define OWL_DMA_MODE_CME                        BIT(31)
  93
  94 /* OWL_DMAX_LINKLIST_CTL Bits */
  95 #define OWL_DMA_LLC_SAV(x)                      (((x) & GENMASK(1, 0)) << 8)
  96 #define OWL_DMA_LLC_SAV_INC                     OWL_DMA_LLC_SAV(0)
  97 #define OWL_DMA_LLC_SAV_LOAD_NEXT               OWL_DMA_LLC_SAV(1)
  98 #define OWL_DMA_LLC_SAV_LOAD_PREV               OWL_DMA_LLC_SAV(2)
  99 #define OWL_DMA_LLC_DAV(x)                      (((x) & GENMASK(1, 0)) << 10)
 100 #define OWL_DMA_LLC_DAV_INC                     OWL_DMA_LLC_DAV(0)
 101 #define OWL_DMA_LLC_DAV_LOAD_NEXT               OWL_DMA_LLC_DAV(1)
 102 #define OWL_DMA_LLC_DAV_LOAD_PREV               OWL_DMA_LLC_DAV(2)
 103 #define OWL_DMA_LLC_SUSPEND                     BIT(16)
 104
 105 /* OWL_DMAX_INT_CTL Bits */
 106 #define OWL_DMA_INTCTL_BLOCK                    BIT(0)
 107 #define OWL_DMA_INTCTL_SUPER_BLOCK              BIT(1)
 108 #define OWL_DMA_INTCTL_FRAME                    BIT(2)
 109 #define OWL_DMA_INTCTL_HALF_FRAME               BIT(3)
 110 #define OWL_DMA_INTCTL_LAST_FRAME               BIT(4)
 111
 112 /* OWL_DMAX_INT_STATUS Bits */
 113 #define OWL_DMA_INTSTAT_BLOCK                   BIT(0)
 114 #define OWL_DMA_INTSTAT_SUPER_BLOCK             BIT(1)
 115 #define OWL_DMA_INTSTAT_FRAME                   BIT(2)
 116 #define OWL_DMA_INTSTAT_HALF_FRAME              BIT(3)
 117 #define OWL_DMA_INTSTAT_LAST_FRAME              BIT(4)
 118
 119 /* Pack shift and newshift in a single word */
 120 #define BIT_FIELD(val, width, shift, newshift)  \
 121                 ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
 122
 123 /**
 124  * struct owl_dma_lli_hw - Hardware link list for dma transfer
 125  * @next_lli: physical address of the next link list
 126  * @saddr: source physical address
 127  * @daddr: destination physical address
 128  * @flen: frame length
 129  * @fcnt: frame count
 130  * @src_stride: source stride
 131  * @dst_stride: destination stride
 132  * @ctrla: dma_mode and linklist ctrl config
 133  * @ctrlb: interrupt config
 134  * @const_num: data for constant fill
 135  */
 136 struct owl_dma_lli_hw {
 137         u32     next_lli;
 138         u32     saddr;
 139         u32     daddr;
 140         u32     flen:20;
 141         u32     fcnt:12;
 142         u32     src_stride;
 143         u32     dst_stride;
 144         u32     ctrla;
 145         u32     ctrlb;
 146         u32     const_num;
 147 };
 148
 149 /**
 150  * struct owl_dma_lli - Link list for dma transfer
 151  * @hw: hardware link list
 152  * @phys: physical address of hardware link list
 153  * @node: node for txd's lli_list
 154  */
 155 struct owl_dma_lli {
 156         struct  owl_dma_lli_hw  hw;
 157         dma_addr_t              phys;
 158         struct list_head        node;
 159 };
 160
 161 /**
 162  * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
 163  * @vd: virtual DMA descriptor
 164  * @lli_list: link list of lli nodes
 165  * @cyclic: flag to indicate cyclic transfers
 166  */
 167 struct owl_dma_txd {
 168         struct virt_dma_desc    vd;
 169         struct list_head        lli_list;
 170         bool                    cyclic;
 171 };
 172
 173 /**
 174  * struct owl_dma_pchan - Holder for the physical channels
 175  * @id: physical index to this channel
 176  * @base: virtual memory base for the dma channel
 177  * @vchan: the virtual channel currently being served by this physical channel
 178  */
 179 struct owl_dma_pchan {
 180         u32                     id;
 181         void __iomem            *base;
 182         struct owl_dma_vchan    *vchan;
 183 };
 184
 185 /**
 186  * struct owl_dma_pchan - Wrapper for DMA ENGINE channel
 187  * @vc: wrappped virtual channel
 188  * @pchan: the physical channel utilized by this channel
 189  * @txd: active transaction on this channel
 190  * @cfg: slave configuration for this channel
 191  * @drq: physical DMA request ID for this channel
 192  */
 193 struct owl_dma_vchan {
 194         struct virt_dma_chan    vc;
 195         struct owl_dma_pchan    *pchan;
 196         struct owl_dma_txd      *txd;
 197         struct dma_slave_config cfg;
 198         u8                      drq;
 199 };
 200
 201 /**
 202  * struct owl_dma - Holder for the Owl DMA controller
 203  * @dma: dma engine for this instance
 204  * @base: virtual memory base for the DMA controller
 205  * @clk: clock for the DMA controller
 206  * @lock: a lock to use when change DMA controller global register
 207  * @lli_pool: a pool for the LLI descriptors
 208  * @irq: interrupt ID for the DMA controller
 209  * @nr_pchans: the number of physical channels
 210  * @pchans: array of data for the physical channels
 211  * @nr_vchans: the number of physical channels
 212  * @vchans: array of data for the physical channels
 213  */
 214 struct owl_dma {
 215         struct dma_device       dma;
 216         void __iomem            *base;
 217         struct clk              *clk;
 218         spinlock_t              lock;
 219         struct dma_pool         *lli_pool;
 220         int                     irq;
 221
 222         unsigned int            nr_pchans;
 223         struct owl_dma_pchan    *pchans;
 224
 225         unsigned int            nr_vchans;
 226         struct owl_dma_vchan    *vchans;
 227 };
 228
 229 static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
 230                          u32 val, bool state)
 231 {
 232         u32 regval;
 233
 234         regval = readl(pchan->base + reg);
 235
 236         if (state)
 237                 regval |= val;
 238         else
 239                 regval &= ~val;
 240
 241         writel(val, pchan->base + reg);
 242 }
 243
 244 static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
 245 {
 246         writel(data, pchan->base + reg);
 247 }
 248
 249 static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
 250 {
 251         return readl(pchan->base + reg);
 252 }
 253
 254 static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
 255 {
 256         u32 regval;
 257
 258         regval = readl(od->base + reg);
 259
 260         if (state)
 261                 regval |= val;
 262         else
 263                 regval &= ~val;
 264
 265         writel(val, od->base + reg);
 266 }
 267
 268 static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
 269 {
 270         writel(data, od->base + reg);
 271 }
 272
 273 static u32 dma_readl(struct owl_dma *od, u32 reg)
 274 {
 275         return readl(od->base + reg);
 276 }
 277
 278 static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
 279 {
 280         return container_of(dd, struct owl_dma, dma);
 281 }
 282
 283 static struct device *chan2dev(struct dma_chan *chan)
 284 {
 285         return &chan->dev->device;
 286 }
 287
 288 static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
 289 {
 290         return container_of(chan, struct owl_dma_vchan, vc.chan);
 291 }
 292
 293 static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
 294 {
 295         return container_of(tx, struct owl_dma_txd, vd.tx);
 296 }
 297
 298 static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
 299 {
 300         u32 ctl;
 301
 302         ctl = BIT_FIELD(mode, 4, 28, 28) |
 303               BIT_FIELD(mode, 8, 16, 20) |
 304               BIT_FIELD(mode, 4, 8, 16) |
 305               BIT_FIELD(mode, 6, 0, 10) |
 306               BIT_FIELD(llc_ctl, 2, 10, 8) |
 307               BIT_FIELD(llc_ctl, 2, 8, 6);
 308
 309         return ctl;
 310 }
 311
 312 static inline u32 llc_hw_ctrlb(u32 int_ctl)
 313 {
 314         u32 ctl;
 315
 316         ctl = BIT_FIELD(int_ctl, 7, 0, 18);
 317
 318         return ctl;
 319 }
 320
 321 static void owl_dma_free_lli(struct owl_dma *od,
 322                              struct owl_dma_lli *lli)
 323 {
 324         list_del(&lli->node);
 325         dma_pool_free(od->lli_pool, lli, lli->phys);
 326 }
 327
 328 static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
 329 {
 330         struct owl_dma_lli *lli;
 331         dma_addr_t phys;
 332
 333         lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
 334         if (!lli)
 335                 return NULL;
 336
 337         INIT_LIST_HEAD(&lli->node);
 338         lli->phys = phys;
 339
 340         return lli;
 341 }
 342
 343 static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
 344                                            struct owl_dma_lli *prev,
 345                                            struct owl_dma_lli *next,
 346                                            bool is_cyclic)
 347 {
 348         if (!is_cyclic)
 349                 list_add_tail(&next->node, &txd->lli_list);
 350
 351         if (prev) {
 352                 prev->hw.next_lli = next->phys;
 353                 prev->hw.ctrla |= llc_hw_ctrla(OWL_DMA_MODE_LME, 0);
 354         }
 355
 356         return next;
 357 }
 358
 359 static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
 360                                   struct owl_dma_lli *lli,
 361                                   dma_addr_t src, dma_addr_t dst,
 362                                   u32 len, enum dma_transfer_direction dir,
 363                                   struct dma_slave_config *sconfig,
 364                                   bool is_cyclic)
 365 {
 366         struct owl_dma_lli_hw *hw = &lli->hw;
 367         u32 mode;
 368
 369         mode = OWL_DMA_MODE_PW(0);
 370
 371         switch (dir) {
 372         case DMA_MEM_TO_MEM:
 373                 mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
 374                         OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
 375                         OWL_DMA_MODE_DAM_INC;
 376
 377                 break;
 378         case DMA_MEM_TO_DEV:
 379                 mode |= OWL_DMA_MODE_TS(vchan->drq)
 380                         | OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV
 381                         | OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST;
 382
 383                 /*
 384                  * Hardware only supports 32bit and 8bit buswidth. Since the
 385                  * default is 32bit, select 8bit only when requested.
 386                  */
 387                 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
 388                         mode |= OWL_DMA_MODE_NDDBW_8BIT;
 389
 390                 break;
 391         case DMA_DEV_TO_MEM:
 392                  mode |= OWL_DMA_MODE_TS(vchan->drq)
 393                         | OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU
 394                         | OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC;
 395
 396                 /*
 397                  * Hardware only supports 32bit and 8bit buswidth. Since the
 398                  * default is 32bit, select 8bit only when requested.
 399                  */
 400                 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
 401                         mode |= OWL_DMA_MODE_NDDBW_8BIT;
 402
 403                 break;
 404         default:
 405                 return -EINVAL;
 406         }
 407
 408         hw->next_lli = 0; /* One link list by default */
 409         hw->saddr = src;
 410         hw->daddr = dst;
 411
 412         hw->fcnt = 1; /* Frame count fixed as 1 */
 413         hw->flen = len; /* Max frame length is 1MB */
 414         hw->src_stride = 0;
 415         hw->dst_stride = 0;
 416         hw->ctrla = llc_hw_ctrla(mode,
 417                                  OWL_DMA_LLC_SAV_LOAD_NEXT |
 418                                  OWL_DMA_LLC_DAV_LOAD_NEXT);
 419
 420         if (is_cyclic)
 421                 hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK);
 422         else
 423                 hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
 424
 425         return 0;
 426 }
 427
 428 static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
 429                                                struct owl_dma_vchan *vchan)
 430 {
 431         struct owl_dma_pchan *pchan = NULL;
 432         unsigned long flags;
 433         int i;
 434
 435         for (i = 0; i < od->nr_pchans; i++) {
 436                 pchan = &od->pchans[i];
 437
 438                 spin_lock_irqsave(&od->lock, flags);
 439                 if (!pchan->vchan) {
 440                         pchan->vchan = vchan;
 441                         spin_unlock_irqrestore(&od->lock, flags);
 442                         break;
 443                 }
 444
 445                 spin_unlock_irqrestore(&od->lock, flags);
 446         }
 447
 448         return pchan;
 449 }
 450
 451 static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
 452 {
 453         unsigned int val;
 454
 455         val = dma_readl(od, OWL_DMA_IDLE_STAT);
 456
 457         return !(val & (1 << pchan->id));
 458 }
 459
 460 static void owl_dma_terminate_pchan(struct owl_dma *od,
 461                                     struct owl_dma_pchan *pchan)
 462 {
 463         unsigned long flags;
 464         u32 irq_pd;
 465
 466         pchan_writel(pchan, OWL_DMAX_START, 0);
 467         pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
 468
 469         spin_lock_irqsave(&od->lock, flags);
 470         dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
 471
 472         irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
 473         if (irq_pd & (1 << pchan->id)) {
 474                 dev_warn(od->dma.dev,
 475                          "terminating pchan %d that still has pending irq\n",
 476                          pchan->id);
 477                 dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
 478         }
 479
 480         pchan->vchan = NULL;
 481
 482         spin_unlock_irqrestore(&od->lock, flags);
 483 }
 484
 485 static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan)
 486 {
 487         pchan_writel(pchan, 1, OWL_DMAX_PAUSE);
 488 }
 489
 490 static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan)
 491 {
 492         pchan_writel(pchan, 0, OWL_DMAX_PAUSE);
 493 }
 494
 495 static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
 496 {
 497         struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
 498         struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
 499         struct owl_dma_pchan *pchan = vchan->pchan;
 500         struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
 501         struct owl_dma_lli *lli;
 502         unsigned long flags;
 503         u32 int_ctl;
 504
 505         list_del(&vd->node);
 506
 507         vchan->txd = txd;
 508
 509         /* Wait for channel inactive */
 510         while (owl_dma_pchan_busy(od, pchan))
 511                 cpu_relax();
 512
 513         lli = list_first_entry(&txd->lli_list,
 514                                struct owl_dma_lli, node);
 515
 516         if (txd->cyclic)
 517                 int_ctl = OWL_DMA_INTCTL_BLOCK;
 518         else
 519                 int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;
 520
 521         pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
 522         pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
 523                      OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
 524         pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
 525         pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
 526
 527         /* Clear IRQ status for this pchan */
 528         pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
 529
 530         spin_lock_irqsave(&od->lock, flags);
 531
 532         dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
 533
 534         spin_unlock_irqrestore(&od->lock, flags);
 535
 536         dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
 537
 538         /* Start DMA transfer for this pchan */
 539         pchan_writel(pchan, OWL_DMAX_START, 0x1);
 540
 541         return 0;
 542 }
 543
 544 static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
 545 {
 546         /* Ensure that the physical channel is stopped */
 547         owl_dma_terminate_pchan(od, vchan->pchan);
 548
 549         vchan->pchan = NULL;
 550 }
 551
 552 static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
 553 {
 554         struct owl_dma *od = dev_id;
 555         struct owl_dma_vchan *vchan;
 556         struct owl_dma_pchan *pchan;
 557         unsigned long pending;
 558         int i;
 559         unsigned int global_irq_pending, chan_irq_pending;
 560
 561         spin_lock(&od->lock);
 562
 563         pending = dma_readl(od, OWL_DMA_IRQ_PD0);
 564
 565         /* Clear IRQ status for each pchan */
 566         for_each_set_bit(i, &pending, od->nr_pchans) {
 567                 pchan = &od->pchans[i];
 568                 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
 569         }
 570
 571         /* Clear pending IRQ */
 572         dma_writel(od, OWL_DMA_IRQ_PD0, pending);
 573
 574         /* Check missed pending IRQ */
 575         for (i = 0; i < od->nr_pchans; i++) {
 576                 pchan = &od->pchans[i];
 577                 chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
 578                                    pchan_readl(pchan, OWL_DMAX_INT_STATUS);
 579
 580                 /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
 581                 dma_readl(od, OWL_DMA_IRQ_PD0);
 582
 583                 global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);
 584
 585                 if (chan_irq_pending && !(global_irq_pending & BIT(i))) {
 586                         dev_dbg(od->dma.dev,
 587                                 "global and channel IRQ pending match err\n");
 588
 589                         /* Clear IRQ status for this pchan */
 590                         pchan_update(pchan, OWL_DMAX_INT_STATUS,
 591                                      0xff, false);
 592
 593                         /* Update global IRQ pending */
 594                         pending |= BIT(i);
 595                 }
 596         }
 597
 598         spin_unlock(&od->lock);
 599
 600         for_each_set_bit(i, &pending, od->nr_pchans) {
 601                 struct owl_dma_txd *txd;
 602
 603                 pchan = &od->pchans[i];
 604
 605                 vchan = pchan->vchan;
 606                 if (!vchan) {
 607                         dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",
 608                                  pchan->id);
 609                         continue;
 610                 }
 611
 612                 spin_lock(&vchan->vc.lock);
 613
 614                 txd = vchan->txd;
 615                 if (txd) {
 616                         vchan->txd = NULL;
 617
 618                         vchan_cookie_complete(&txd->vd);
 619
 620                         /*
 621                          * Start the next descriptor (if any),
 622                          * otherwise free this channel.
 623                          */
 624                         if (vchan_next_desc(&vchan->vc))
 625                                 owl_dma_start_next_txd(vchan);
 626                         else
 627                                 owl_dma_phy_free(od, vchan);
 628                 }
 629
 630                 spin_unlock(&vchan->vc.lock);
 631         }
 632
 633         return IRQ_HANDLED;
 634 }
 635
 636 static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
 637 {
 638         struct owl_dma_lli *lli, *_lli;
 639
 640         if (unlikely(!txd))
 641                 return;
 642
 643         list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
 644                 owl_dma_free_lli(od, lli);
 645
 646         kfree(txd);
 647 }
 648
 649 static void owl_dma_desc_free(struct virt_dma_desc *vd)
 650 {
 651         struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
 652         struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
 653
 654         owl_dma_free_txd(od, txd);
 655 }
 656
 657 static int owl_dma_terminate_all(struct dma_chan *chan)
 658 {
 659         struct owl_dma *od = to_owl_dma(chan->device);
 660         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 661         unsigned long flags;
 662         LIST_HEAD(head);
 663
 664         spin_lock_irqsave(&vchan->vc.lock, flags);
 665
 666         if (vchan->pchan)
 667                 owl_dma_phy_free(od, vchan);
 668
 669         if (vchan->txd) {
 670                 owl_dma_desc_free(&vchan->txd->vd);
 671                 vchan->txd = NULL;
 672         }
 673
 674         vchan_get_all_descriptors(&vchan->vc, &head);
 675         vchan_dma_desc_free_list(&vchan->vc, &head);
 676
 677         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 678
 679         return 0;
 680 }
 681
 682 static int owl_dma_config(struct dma_chan *chan,
 683                           struct dma_slave_config *config)
 684 {
 685         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 686
 687         /* Reject definitely invalid configurations */
 688         if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 689             config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 690                 return -EINVAL;
 691
 692         memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config));
 693
 694         return 0;
 695 }
 696
 697 static int owl_dma_pause(struct dma_chan *chan)
 698 {
 699         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 700         unsigned long flags;
 701
 702         spin_lock_irqsave(&vchan->vc.lock, flags);
 703
 704         owl_dma_pause_pchan(vchan->pchan);
 705
 706         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 707
 708         return 0;
 709 }
 710
 711 static int owl_dma_resume(struct dma_chan *chan)
 712 {
 713         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 714         unsigned long flags;
 715
 716         if (!vchan->pchan && !vchan->txd)
 717                 return 0;
 718
 719         dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
 720
 721         spin_lock_irqsave(&vchan->vc.lock, flags);
 722
 723         owl_dma_resume_pchan(vchan->pchan);
 724
 725         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 726
 727         return 0;
 728 }
 729
 730 static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
 731 {
 732         struct owl_dma_pchan *pchan;
 733         struct owl_dma_txd *txd;
 734         struct owl_dma_lli *lli;
 735         unsigned int next_lli_phy;
 736         size_t bytes;
 737
 738         pchan = vchan->pchan;
 739         txd = vchan->txd;
 740
 741         if (!pchan || !txd)
 742                 return 0;
 743
 744         /* Get remain count of current node in link list */
 745         bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
 746
 747         /* Loop through the preceding nodes to get total remaining bytes */
 748         if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
 749                 next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
 750                 list_for_each_entry(lli, &txd->lli_list, node) {
 751                         /* Start from the next active node */
 752                         if (lli->phys == next_lli_phy) {
 753                                 list_for_each_entry(lli, &txd->lli_list, node)
 754                                         bytes += lli->hw.flen;
 755                                 break;
 756                         }
 757                 }
 758         }
 759
 760         return bytes;
 761 }
 762
 763 static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
 764                                          dma_cookie_t cookie,
 765                                          struct dma_tx_state *state)
 766 {
 767         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 768         struct owl_dma_lli *lli;
 769         struct virt_dma_desc *vd;
 770         struct owl_dma_txd *txd;
 771         enum dma_status ret;
 772         unsigned long flags;
 773         size_t bytes = 0;
 774
 775         ret = dma_cookie_status(chan, cookie, state);
 776         if (ret == DMA_COMPLETE || !state)
 777                 return ret;
 778
 779         spin_lock_irqsave(&vchan->vc.lock, flags);
 780
 781         vd = vchan_find_desc(&vchan->vc, cookie);
 782         if (vd) {
 783                 txd = to_owl_txd(&vd->tx);
 784                 list_for_each_entry(lli, &txd->lli_list, node)
 785                         bytes += lli->hw.flen;
 786         } else {
 787                 bytes = owl_dma_getbytes_chan(vchan);
 788         }
 789
 790         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 791
 792         dma_set_residue(state, bytes);
 793
 794         return ret;
 795 }
 796
 797 static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
 798 {
 799         struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
 800         struct owl_dma_pchan *pchan;
 801
 802         pchan = owl_dma_get_pchan(od, vchan);
 803         if (!pchan)
 804                 return;
 805
 806         dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);
 807
 808         vchan->pchan = pchan;
 809         owl_dma_start_next_txd(vchan);
 810 }
 811
 812 static void owl_dma_issue_pending(struct dma_chan *chan)
 813 {
 814         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 815         unsigned long flags;
 816
 817         spin_lock_irqsave(&vchan->vc.lock, flags);
 818         if (vchan_issue_pending(&vchan->vc)) {
 819                 if (!vchan->pchan)
 820                         owl_dma_phy_alloc_and_start(vchan);
 821         }
 822         spin_unlock_irqrestore(&vchan->vc.lock, flags);
 823 }
 824
 825 static struct dma_async_tx_descriptor
 826                 *owl_dma_prep_memcpy(struct dma_chan *chan,
 827                                      dma_addr_t dst, dma_addr_t src,
 828                                      size_t len, unsigned long flags)
 829 {
 830         struct owl_dma *od = to_owl_dma(chan->device);
 831         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 832         struct owl_dma_txd *txd;
 833         struct owl_dma_lli *lli, *prev = NULL;
 834         size_t offset, bytes;
 835         int ret;
 836
 837         if (!len)
 838                 return NULL;
 839
 840         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 841         if (!txd)
 842                 return NULL;
 843
 844         INIT_LIST_HEAD(&txd->lli_list);
 845
 846         /* Process the transfer as frame by frame */
 847         for (offset = 0; offset < len; offset += bytes) {
 848                 lli = owl_dma_alloc_lli(od);
 849                 if (!lli) {
 850                         dev_warn(chan2dev(chan), "failed to allocate lli\n");
 851                         goto err_txd_free;
 852                 }
 853
 854                 bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
 855
 856                 ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,
 857                                       bytes, DMA_MEM_TO_MEM,
 858                                       &vchan->cfg, txd->cyclic);
 859                 if (ret) {
 860                         dev_warn(chan2dev(chan), "failed to config lli\n");
 861                         goto err_txd_free;
 862                 }
 863
 864                 prev = owl_dma_add_lli(txd, prev, lli, false);
 865         }
 866
 867         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 868
 869 err_txd_free:
 870         owl_dma_free_txd(od, txd);
 871         return NULL;
 872 }
 873
 874 static struct dma_async_tx_descriptor
 875                 *owl_dma_prep_slave_sg(struct dma_chan *chan,
 876                                        struct scatterlist *sgl,
 877                                        unsigned int sg_len,
 878                                        enum dma_transfer_direction dir,
 879                                        unsigned long flags, void *context)
 880 {
 881         struct owl_dma *od = to_owl_dma(chan->device);
 882         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 883         struct dma_slave_config *sconfig = &vchan->cfg;
 884         struct owl_dma_txd *txd;
 885         struct owl_dma_lli *lli, *prev = NULL;
 886         struct scatterlist *sg;
 887         dma_addr_t addr, src = 0, dst = 0;
 888         size_t len;
 889         int ret, i;
 890
 891         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 892         if (!txd)
 893                 return NULL;
 894
 895         INIT_LIST_HEAD(&txd->lli_list);
 896
 897         for_each_sg(sgl, sg, sg_len, i) {
 898                 addr = sg_dma_address(sg);
 899                 len = sg_dma_len(sg);
 900
 901                 if (len > OWL_DMA_FRAME_MAX_LENGTH) {
 902                         dev_err(od->dma.dev,
 903                                 "frame length exceeds max supported length");
 904                         goto err_txd_free;
 905                 }
 906
 907                 lli = owl_dma_alloc_lli(od);
 908                 if (!lli) {
 909                         dev_err(chan2dev(chan), "failed to allocate lli");
 910                         goto err_txd_free;
 911                 }
 912
 913                 if (dir == DMA_MEM_TO_DEV) {
 914                         src = addr;
 915                         dst = sconfig->dst_addr;
 916                 } else {
 917                         src = sconfig->src_addr;
 918                         dst = addr;
 919                 }
 920
 921                 ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,
 922                                       txd->cyclic);
 923                 if (ret) {
 924                         dev_warn(chan2dev(chan), "failed to config lli");
 925                         goto err_txd_free;
 926                 }
 927
 928                 prev = owl_dma_add_lli(txd, prev, lli, false);
 929         }
 930
 931         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 932
 933 err_txd_free:
 934         owl_dma_free_txd(od, txd);
 935
 936         return NULL;
 937 }
 938
 939 static struct dma_async_tx_descriptor
 940                 *owl_prep_dma_cyclic(struct dma_chan *chan,
 941                                      dma_addr_t buf_addr, size_t buf_len,
 942                                      size_t period_len,
 943                                      enum dma_transfer_direction dir,
 944                                      unsigned long flags)
 945 {
 946         struct owl_dma *od = to_owl_dma(chan->device);
 947         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
 948         struct dma_slave_config *sconfig = &vchan->cfg;
 949         struct owl_dma_txd *txd;
 950         struct owl_dma_lli *lli, *prev = NULL, *first = NULL;
 951         dma_addr_t src = 0, dst = 0;
 952         unsigned int periods = buf_len / period_len;
 953         int ret, i;
 954
 955         txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 956         if (!txd)
 957                 return NULL;
 958
 959         INIT_LIST_HEAD(&txd->lli_list);
 960         txd->cyclic = true;
 961
 962         for (i = 0; i < periods; i++) {
 963                 lli = owl_dma_alloc_lli(od);
 964                 if (!lli) {
 965                         dev_warn(chan2dev(chan), "failed to allocate lli");
 966                         goto err_txd_free;
 967                 }
 968
 969                 if (dir == DMA_MEM_TO_DEV) {
 970                         src = buf_addr + (period_len * i);
 971                         dst = sconfig->dst_addr;
 972                 } else if (dir == DMA_DEV_TO_MEM) {
 973                         src = sconfig->src_addr;
 974                         dst = buf_addr + (period_len * i);
 975                 }
 976
 977                 ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,
 978                                       dir, sconfig, txd->cyclic);
 979                 if (ret) {
 980                         dev_warn(chan2dev(chan), "failed to config lli");
 981                         goto err_txd_free;
 982                 }
 983
 984                 if (!first)
 985                         first = lli;
 986
 987                 prev = owl_dma_add_lli(txd, prev, lli, false);
 988         }
 989
 990         /* close the cyclic list */
 991         owl_dma_add_lli(txd, prev, first, true);
 992
 993         return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 994
 995 err_txd_free:
 996         owl_dma_free_txd(od, txd);
 997
 998         return NULL;
 999 }
1000
1001 static void owl_dma_free_chan_resources(struct dma_chan *chan)
1002 {
1003         struct owl_dma_vchan *vchan = to_owl_vchan(chan);
1004
1005         /* Ensure all queued descriptors are freed */
1006         vchan_free_chan_resources(&vchan->vc);
1007 }
1008
1009 static inline void owl_dma_free(struct owl_dma *od)
1010 {
1011         struct owl_dma_vchan *vchan = NULL;
1012         struct owl_dma_vchan *next;
1013
1014         list_for_each_entry_safe(vchan,
1015                                  next, &od->dma.channels, vc.chan.device_node) {
1016                 list_del(&vchan->vc.chan.device_node);
1017                 tasklet_kill(&vchan->vc.task);
1018         }
1019 }
1020
1021 static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec,
1022                                          struct of_dma *ofdma)
1023 {
1024         struct owl_dma *od = ofdma->of_dma_data;
1025         struct owl_dma_vchan *vchan;
1026         struct dma_chan *chan;
1027         u8 drq = dma_spec->args[0];
1028
1029         if (drq > od->nr_vchans)
1030                 return NULL;
1031
1032         chan = dma_get_any_slave_channel(&od->dma);
1033         if (!chan)
1034                 return NULL;
1035
1036         vchan = to_owl_vchan(chan);
1037         vchan->drq = drq;
1038
1039         return chan;
1040 }
1041
1042 static int owl_dma_probe(struct platform_device *pdev)
1043 {
1044         struct device_node *np = pdev->dev.of_node;
1045         struct owl_dma *od;
1046         struct resource *res;
1047         int ret, i, nr_channels, nr_requests;
1048
1049         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1050         if (!od)
1051                 return -ENOMEM;
1052
1053         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1054         if (!res)
1055                 return -EINVAL;
1056
1057         od->base = devm_ioremap_resource(&pdev->dev, res);
1058         if (IS_ERR(od->base))
1059                 return PTR_ERR(od->base);
1060
1061         ret = of_property_read_u32(np, "dma-channels", &nr_channels);
1062         if (ret) {
1063                 dev_err(&pdev->dev, "can't get dma-channels\n");
1064                 return ret;
1065         }
1066
1067         ret = of_property_read_u32(np, "dma-requests", &nr_requests);
1068         if (ret) {
1069                 dev_err(&pdev->dev, "can't get dma-requests\n");
1070                 return ret;
1071         }
1072
1073         dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",
1074                  nr_channels, nr_requests);
1075
1076         od->nr_pchans = nr_channels;
1077         od->nr_vchans = nr_requests;
1078
1079         pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
1080
1081         platform_set_drvdata(pdev, od);
1082         spin_lock_init(&od->lock);
1083
1084         dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);
1085         dma_cap_set(DMA_SLAVE, od->dma.cap_mask);
1086         dma_cap_set(DMA_CYCLIC, od->dma.cap_mask);
1087
1088         od->dma.dev = &pdev->dev;
1089         od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
1090         od->dma.device_tx_status = owl_dma_tx_status;
1091         od->dma.device_issue_pending = owl_dma_issue_pending;
1092         od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;
1093         od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg;
1094         od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic;
1095         od->dma.device_config = owl_dma_config;
1096         od->dma.device_pause = owl_dma_pause;
1097         od->dma.device_resume = owl_dma_resume;
1098         od->dma.device_terminate_all = owl_dma_terminate_all;
1099         od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1100         od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1101         od->dma.directions = BIT(DMA_MEM_TO_MEM);
1102         od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1103
1104         INIT_LIST_HEAD(&od->dma.channels);
1105
1106         od->clk = devm_clk_get(&pdev->dev, NULL);
1107         if (IS_ERR(od->clk)) {
1108                 dev_err(&pdev->dev, "unable to get clock\n");
1109                 return PTR_ERR(od->clk);
1110         }
1111
1112         /*
1113          * Eventhough the DMA controller is capable of generating 4
1114          * IRQ's for DMA priority feature, we only use 1 IRQ for
1115          * simplification.
1116          */
1117         od->irq = platform_get_irq(pdev, 0);
1118         ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
1119                                dev_name(&pdev->dev), od);
1120         if (ret) {
1121                 dev_err(&pdev->dev, "unable to request IRQ\n");
1122                 return ret;
1123         }
1124
1125         /* Init physical channel */
1126         od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
1127                                   sizeof(struct owl_dma_pchan), GFP_KERNEL);
1128         if (!od->pchans)
1129                 return -ENOMEM;
1130
1131         for (i = 0; i < od->nr_pchans; i++) {
1132                 struct owl_dma_pchan *pchan = &od->pchans[i];
1133
1134                 pchan->id = i;
1135                 pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
1136         }
1137
1138         /* Init virtual channel */
1139         od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
1140                                   sizeof(struct owl_dma_vchan), GFP_KERNEL);
1141         if (!od->vchans)
1142                 return -ENOMEM;
1143
1144         for (i = 0; i < od->nr_vchans; i++) {
1145                 struct owl_dma_vchan *vchan = &od->vchans[i];
1146
1147                 vchan->vc.desc_free = owl_dma_desc_free;
1148                 vchan_init(&vchan->vc, &od->dma);
1149         }
1150
1151         /* Create a pool of consistent memory blocks for hardware descriptors */
1152         od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
1153                                        sizeof(struct owl_dma_lli),
1154                                        __alignof__(struct owl_dma_lli),
1155                                        0);
1156         if (!od->lli_pool) {
1157                 dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");
1158                 return -ENOMEM;
1159         }
1160
1161         clk_prepare_enable(od->clk);
1162
1163         ret = dma_async_device_register(&od->dma);
1164         if (ret) {
1165                 dev_err(&pdev->dev, "failed to register DMA engine device\n");
1166                 goto err_pool_free;
1167         }
1168
1169         /* Device-tree DMA controller registration */
1170         ret = of_dma_controller_register(pdev->dev.of_node,
1171                                          owl_dma_of_xlate, od);
1172         if (ret) {
1173                 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1174                 goto err_dma_unregister;
1175         }
1176
1177         return 0;
1178
1179 err_dma_unregister:
1180         dma_async_device_unregister(&od->dma);
1181 err_pool_free:
1182         clk_disable_unprepare(od->clk);
1183         dma_pool_destroy(od->lli_pool);
1184
1185         return ret;
1186 }
1187
1188 static int owl_dma_remove(struct platform_device *pdev)
1189 {
1190         struct owl_dma *od = platform_get_drvdata(pdev);
1191
1192         of_dma_controller_free(pdev->dev.of_node);
1193         dma_async_device_unregister(&od->dma);
1194
1195         /* Mask all interrupts for this execution environment */
1196         dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
1197
1198         /* Make sure we won't have any further interrupts */
1199         devm_free_irq(od->dma.dev, od->irq, od);
1200
1201         owl_dma_free(od);
1202
1203         clk_disable_unprepare(od->clk);
1204
1205         return 0;
1206 }
1207
1208 static const struct of_device_id owl_dma_match[] = {
1209         { .compatible = "actions,s900-dma", },
1210         { /* sentinel */ }
1211 };
1212 MODULE_DEVICE_TABLE(of, owl_dma_match);
1213
1214 static struct platform_driver owl_dma_driver = {
1215         .probe  = owl_dma_probe,
1216         .remove = owl_dma_remove,
1217         .driver = {
1218                 .name = "dma-owl",
1219                 .of_match_table = of_match_ptr(owl_dma_match),
1220         },
1221 };
1222
1223 static int owl_dma_init(void)
1224 {
1225         return platform_driver_register(&owl_dma_driver);
1226 }
1227 subsys_initcall(owl_dma_init);
1228
1229 static void __exit owl_dma_exit(void)
1230 {
1231         platform_driver_unregister(&owl_dma_driver);
1232 }
1233 module_exit(owl_dma_exit);
1234
1235 MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
1236 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
1237 MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
1238 MODULE_LICENSE("GPL");