1 // SPDX-License-Identifier: GPL-2.0+
3 * Copyright (C) 2018 Álvaro Fernández Rojas <noltari@gmail.com>
5 * Derived from linux/drivers/dma/bcm63xx-iudma.c:
6 * Copyright (C) 2015 Simon Arlott <simon@fire.lp0.eu>
8 * Derived from linux/drivers/net/ethernet/broadcom/bcm63xx_enet.c:
9 * Copyright (C) 2008 Maxime Bizon <mbizon@freebox.fr>
11 * Derived from bcm963xx_4.12L.06B_consumer/shared/opensource/include/bcm963xx/63268_map_part.h:
12 * Copyright (C) 2000-2010 Broadcom Corporation
14 * Derived from bcm963xx_4.12L.06B_consumer/bcmdrivers/opensource/net/enet/impl4/bcmenet.c:
15 * Copyright (C) 2010 Broadcom Corporation
22 #include <dma-uclass.h>
31 #define DMA_CHAN_FLOWC(x) ((x) >> 1)
32 #define DMA_CHAN_MAX 16
33 #define DMA_CHAN_SIZE 0x10
34 #define DMA_CHAN_TOUT 500
36 /* DMA Global Configuration register */
37 #define DMA_CFG_REG 0x00
38 #define DMA_CFG_ENABLE_SHIFT 0
39 #define DMA_CFG_ENABLE_MASK (1 << DMA_CFG_ENABLE_SHIFT)
40 #define DMA_CFG_FLOWC_ENABLE(x) BIT(DMA_CHAN_FLOWC(x) + 1)
41 #define DMA_CFG_NCHANS_SHIFT 24
42 #define DMA_CFG_NCHANS_MASK (0xf << DMA_CFG_NCHANS_SHIFT)
44 /* DMA Global Flow Control registers */
45 #define DMA_FLOWC_THR_LO_REG(x) (0x04 + DMA_CHAN_FLOWC(x) * 0x0c)
46 #define DMA_FLOWC_THR_HI_REG(x) (0x08 + DMA_CHAN_FLOWC(x) * 0x0c)
47 #define DMA_FLOWC_ALLOC_REG(x) (0x0c + DMA_CHAN_FLOWC(x) * 0x0c)
48 #define DMA_FLOWC_ALLOC_FORCE_SHIFT 31
49 #define DMA_FLOWC_ALLOC_FORCE_MASK (1 << DMA_FLOWC_ALLOC_FORCE_SHIFT)
51 /* DMA Global Reset register */
52 #define DMA_RST_REG 0x34
53 #define DMA_RST_CHAN_SHIFT 0
54 #define DMA_RST_CHAN_MASK(x) (1 << x)
56 /* DMA Channel Configuration register */
57 #define DMAC_CFG_REG(x) (DMA_CHAN_SIZE * (x) + 0x00)
58 #define DMAC_CFG_ENABLE_SHIFT 0
59 #define DMAC_CFG_ENABLE_MASK (1 << DMAC_CFG_ENABLE_SHIFT)
60 #define DMAC_CFG_PKT_HALT_SHIFT 1
61 #define DMAC_CFG_PKT_HALT_MASK (1 << DMAC_CFG_PKT_HALT_SHIFT)
62 #define DMAC_CFG_BRST_HALT_SHIFT 2
63 #define DMAC_CFG_BRST_HALT_MASK (1 << DMAC_CFG_BRST_HALT_SHIFT)
65 /* DMA Channel Max Burst Length register */
66 #define DMAC_BURST_REG(x) (DMA_CHAN_SIZE * (x) + 0x0c)
68 /* DMA SRAM Descriptor Ring Start register */
69 #define DMAS_RSTART_REG(x) (DMA_CHAN_SIZE * (x) + 0x00)
71 /* DMA SRAM State/Bytes done/ring offset register */
72 #define DMAS_STATE_DATA_REG(x) (DMA_CHAN_SIZE * (x) + 0x04)
74 /* DMA SRAM Buffer Descriptor status and length register */
75 #define DMAS_DESC_LEN_STATUS_REG(x) (DMA_CHAN_SIZE * (x) + 0x08)
77 /* DMA SRAM Buffer Descriptor status and length register */
78 #define DMAS_DESC_BASE_BUFPTR_REG(x) (DMA_CHAN_SIZE * (x) + 0x0c)
80 /* DMA Descriptor Status */
81 #define DMAD_ST_CRC_SHIFT 8
82 #define DMAD_ST_CRC_MASK (1 << DMAD_ST_CRC_SHIFT)
83 #define DMAD_ST_WRAP_SHIFT 12
84 #define DMAD_ST_WRAP_MASK (1 << DMAD_ST_WRAP_SHIFT)
85 #define DMAD_ST_SOP_SHIFT 13
86 #define DMAD_ST_SOP_MASK (1 << DMAD_ST_SOP_SHIFT)
87 #define DMAD_ST_EOP_SHIFT 14
88 #define DMAD_ST_EOP_MASK (1 << DMAD_ST_EOP_SHIFT)
89 #define DMAD_ST_OWN_SHIFT 15
90 #define DMAD_ST_OWN_MASK (1 << DMAD_ST_OWN_SHIFT)
92 #define DMAD6348_ST_OV_ERR_SHIFT 0
93 #define DMAD6348_ST_OV_ERR_MASK (1 << DMAD6348_ST_OV_ERR_SHIFT)
94 #define DMAD6348_ST_CRC_ERR_SHIFT 1
95 #define DMAD6348_ST_CRC_ERR_MASK (1 << DMAD6348_ST_CRC_ERR_SHIFT)
96 #define DMAD6348_ST_RX_ERR_SHIFT 2
97 #define DMAD6348_ST_RX_ERR_MASK (1 << DMAD6348_ST_RX_ERR_SHIFT)
98 #define DMAD6348_ST_OS_ERR_SHIFT 4
99 #define DMAD6348_ST_OS_ERR_MASK (1 << DMAD6348_ST_OS_ERR_SHIFT)
100 #define DMAD6348_ST_UN_ERR_SHIFT 9
101 #define DMAD6348_ST_UN_ERR_MASK (1 << DMAD6348_ST_UN_ERR_SHIFT)
103 struct bcm6348_dma_desc {
109 struct bcm6348_chan_priv {
110 void __iomem *dma_ring;
111 uint8_t dma_ring_size;
118 struct bcm6348_iudma_hw {
122 struct bcm6348_iudma_priv {
123 const struct bcm6348_iudma_hw *hw;
127 struct bcm6348_chan_priv **ch_priv;
131 static inline bool bcm6348_iudma_chan_is_rx(uint8_t ch)
136 static inline void bcm6348_iudma_fdc(void *ptr, ulong size)
138 ulong start = (ulong) ptr;
140 flush_dcache_range(start, start + size);
143 static inline void bcm6348_iudma_idc(void *ptr, ulong size)
145 ulong start = (ulong) ptr;
147 invalidate_dcache_range(start, start + size);
150 static void bcm6348_iudma_chan_stop(struct bcm6348_iudma_priv *priv,
153 unsigned int timeout = DMA_CHAN_TOUT;
158 if (timeout > DMA_CHAN_TOUT / 2)
159 halt = DMAC_CFG_PKT_HALT_MASK;
161 halt = DMAC_CFG_BRST_HALT_MASK;
163 /* try to stop dma channel */
164 writel_be(halt, priv->chan + DMAC_CFG_REG(ch));
167 /* check if channel was stopped */
168 cfg = readl_be(priv->chan + DMAC_CFG_REG(ch));
169 if (!(cfg & DMAC_CFG_ENABLE_MASK))
176 pr_err("unable to stop channel %u\n", ch);
178 /* reset dma channel */
179 setbits_be32(priv->base + DMA_RST_REG, DMA_RST_CHAN_MASK(ch));
181 clrbits_be32(priv->base + DMA_RST_REG, DMA_RST_CHAN_MASK(ch));
184 static int bcm6348_iudma_disable(struct dma *dma)
186 struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
187 struct bcm6348_chan_priv *ch_priv = priv->ch_priv[dma->id];
189 /* stop dma channel */
190 bcm6348_iudma_chan_stop(priv, dma->id);
192 /* dma flow control */
193 if (bcm6348_iudma_chan_is_rx(dma->id))
194 writel_be(DMA_FLOWC_ALLOC_FORCE_MASK,
195 DMA_FLOWC_ALLOC_REG(dma->id));
197 /* init channel config */
198 ch_priv->running = false;
199 ch_priv->desc_id = 0;
200 if (bcm6348_iudma_chan_is_rx(dma->id))
201 ch_priv->desc_cnt = 0;
203 ch_priv->desc_cnt = ch_priv->dma_ring_size;
208 static int bcm6348_iudma_enable(struct dma *dma)
210 const struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
211 struct bcm6348_chan_priv *ch_priv = priv->ch_priv[dma->id];
212 struct bcm6348_dma_desc *dma_desc = ch_priv->dma_ring;
216 for (i = 0; i < ch_priv->desc_cnt; i++) {
217 if (bcm6348_iudma_chan_is_rx(dma->id)) {
218 ch_priv->busy_desc[i] = false;
219 dma_desc->status |= DMAD_ST_OWN_MASK;
221 dma_desc->status = 0;
222 dma_desc->length = 0;
223 dma_desc->address = 0;
226 if (i == ch_priv->desc_cnt - 1)
227 dma_desc->status |= DMAD_ST_WRAP_MASK;
232 /* init to first descriptor */
233 ch_priv->desc_id = 0;
235 /* force cache writeback */
236 bcm6348_iudma_fdc(ch_priv->dma_ring,
237 sizeof(*dma_desc) * ch_priv->desc_cnt);
240 writel_be(0, priv->sram + DMAS_STATE_DATA_REG(dma->id));
241 writel_be(0, priv->sram + DMAS_DESC_LEN_STATUS_REG(dma->id));
242 writel_be(0, priv->sram + DMAS_DESC_BASE_BUFPTR_REG(dma->id));
244 /* set dma ring start */
245 writel_be(virt_to_phys(ch_priv->dma_ring),
246 priv->sram + DMAS_RSTART_REG(dma->id));
248 /* set flow control */
249 if (bcm6348_iudma_chan_is_rx(dma->id)) {
252 setbits_be32(priv->base + DMA_CFG_REG,
253 DMA_CFG_FLOWC_ENABLE(dma->id));
255 val = ch_priv->desc_cnt / 3;
256 writel_be(val, priv->base + DMA_FLOWC_THR_LO_REG(dma->id));
258 val = (ch_priv->desc_cnt * 2) / 3;
259 writel_be(val, priv->base + DMA_FLOWC_THR_HI_REG(dma->id));
261 writel_be(0, priv->base + DMA_FLOWC_ALLOC_REG(dma->id));
264 /* set dma max burst */
265 writel_be(ch_priv->desc_cnt,
266 priv->chan + DMAC_BURST_REG(dma->id));
268 /* kick rx dma channel */
269 if (bcm6348_iudma_chan_is_rx(dma->id))
270 setbits_be32(priv->chan + DMAC_CFG_REG(dma->id),
271 DMAC_CFG_ENABLE_MASK);
273 /* channel is now enabled */
274 ch_priv->running = true;
279 static int bcm6348_iudma_request(struct dma *dma)
281 const struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
282 struct bcm6348_chan_priv *ch_priv;
284 /* check if channel is valid */
285 if (dma->id >= priv->n_channels)
288 /* alloc channel private data */
289 priv->ch_priv[dma->id] = calloc(1, sizeof(struct bcm6348_chan_priv));
290 if (!priv->ch_priv[dma->id])
292 ch_priv = priv->ch_priv[dma->id];
295 if (bcm6348_iudma_chan_is_rx(dma->id))
296 ch_priv->dma_ring_size = DMA_RX_DESC;
298 ch_priv->dma_ring_size = DMA_TX_DESC;
301 malloc_cache_aligned(sizeof(struct bcm6348_dma_desc) *
302 ch_priv->dma_ring_size);
303 if (!ch_priv->dma_ring)
306 /* init channel config */
307 ch_priv->running = false;
308 ch_priv->desc_id = 0;
309 if (bcm6348_iudma_chan_is_rx(dma->id)) {
310 ch_priv->desc_cnt = 0;
311 ch_priv->busy_desc = calloc(ch_priv->desc_cnt, sizeof(bool));
313 ch_priv->desc_cnt = ch_priv->dma_ring_size;
314 ch_priv->busy_desc = NULL;
320 static int bcm6348_iudma_receive(struct dma *dma, void **dst, void *metadata)
322 const struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
323 const struct bcm6348_iudma_hw *hw = priv->hw;
324 struct bcm6348_chan_priv *ch_priv = priv->ch_priv[dma->id];
325 struct bcm6348_dma_desc *dma_desc = dma_desc = ch_priv->dma_ring;
328 if (!ch_priv->running)
331 /* get dma ring descriptor address */
332 dma_desc += ch_priv->desc_id;
334 /* invalidate cache data */
335 bcm6348_iudma_idc(dma_desc, sizeof(*dma_desc));
338 if (dma_desc->status & DMAD_ST_OWN_MASK)
342 if (!(dma_desc->status & DMAD_ST_EOP_MASK) ||
343 !(dma_desc->status & DMAD_ST_SOP_MASK) ||
344 (dma_desc->status & hw->err_mask)) {
345 pr_err("invalid pkt received (ch=%ld desc=%u) (st=%04x)\n",
346 dma->id, ch_priv->desc_id, dma_desc->status);
349 /* set dma buffer address */
350 *dst = phys_to_virt(dma_desc->address);
352 /* invalidate cache data */
353 bcm6348_iudma_idc(*dst, dma_desc->length);
355 /* return packet length */
356 ret = dma_desc->length;
359 /* busy dma descriptor */
360 ch_priv->busy_desc[ch_priv->desc_id] = true;
362 /* increment dma descriptor */
363 ch_priv->desc_id = (ch_priv->desc_id + 1) % ch_priv->desc_cnt;
368 static int bcm6348_iudma_send(struct dma *dma, void *src, size_t len,
371 const struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
372 struct bcm6348_chan_priv *ch_priv = priv->ch_priv[dma->id];
373 struct bcm6348_dma_desc *dma_desc;
376 if (!ch_priv->running)
380 bcm6348_iudma_fdc(src, len);
382 /* get dma ring descriptor address */
383 dma_desc = ch_priv->dma_ring;
384 dma_desc += ch_priv->desc_id;
386 /* config dma descriptor */
387 status = (DMAD_ST_OWN_MASK |
391 if (ch_priv->desc_id == ch_priv->desc_cnt - 1)
392 status |= DMAD_ST_WRAP_MASK;
394 /* set dma descriptor */
395 dma_desc->address = virt_to_phys(src);
396 dma_desc->length = len;
397 dma_desc->status = status;
400 bcm6348_iudma_fdc(dma_desc, sizeof(*dma_desc));
402 /* kick tx dma channel */
403 setbits_be32(priv->chan + DMAC_CFG_REG(dma->id), DMAC_CFG_ENABLE_MASK);
405 /* poll dma status */
407 /* invalidate cache */
408 bcm6348_iudma_idc(dma_desc, sizeof(*dma_desc));
410 if (!(dma_desc->status & DMAD_ST_OWN_MASK))
414 /* increment dma descriptor */
415 ch_priv->desc_id = (ch_priv->desc_id + 1) % ch_priv->desc_cnt;
420 static int bcm6348_iudma_free_rcv_buf(struct dma *dma, void *dst, size_t size)
422 const struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
423 struct bcm6348_chan_priv *ch_priv = priv->ch_priv[dma->id];
424 struct bcm6348_dma_desc *dma_desc = ch_priv->dma_ring;
429 /* get dirty dma descriptor */
430 for (i = 0; i < ch_priv->desc_cnt; i++) {
431 if (phys_to_virt(dma_desc->address) == dst)
437 /* dma descriptor not found */
438 if (i == ch_priv->desc_cnt) {
439 pr_err("dirty dma descriptor not found\n");
443 /* invalidate cache */
444 bcm6348_iudma_idc(ch_priv->dma_ring,
445 sizeof(*dma_desc) * ch_priv->desc_cnt);
447 /* free dma descriptor */
448 ch_priv->busy_desc[i] = false;
450 status = DMAD_ST_OWN_MASK;
451 if (i == ch_priv->desc_cnt - 1)
452 status |= DMAD_ST_WRAP_MASK;
454 dma_desc->status |= status;
455 dma_desc->length = PKTSIZE_ALIGN;
457 /* tell dma we allocated one buffer */
458 writel_be(1, DMA_FLOWC_ALLOC_REG(dma->id));
461 bcm6348_iudma_fdc(ch_priv->dma_ring,
462 sizeof(*dma_desc) * ch_priv->desc_cnt);
464 /* kick rx dma channel if disabled */
465 cfg = readl_be(priv->chan + DMAC_CFG_REG(dma->id));
466 if (!(cfg & DMAC_CFG_ENABLE_MASK))
467 setbits_be32(priv->chan + DMAC_CFG_REG(dma->id),
468 DMAC_CFG_ENABLE_MASK);
473 static int bcm6348_iudma_add_rcv_buf(struct dma *dma, void *dst, size_t size)
475 const struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
476 struct bcm6348_chan_priv *ch_priv = priv->ch_priv[dma->id];
477 struct bcm6348_dma_desc *dma_desc = ch_priv->dma_ring;
479 /* no more dma descriptors available */
480 if (ch_priv->desc_cnt == ch_priv->dma_ring_size) {
481 pr_err("max number of buffers reached\n");
485 /* get next dma descriptor */
486 dma_desc += ch_priv->desc_cnt;
488 /* init dma descriptor */
489 dma_desc->address = virt_to_phys(dst);
490 dma_desc->length = size;
491 dma_desc->status = 0;
494 bcm6348_iudma_fdc(dma_desc, sizeof(*dma_desc));
496 /* increment dma descriptors */
502 static int bcm6348_iudma_prepare_rcv_buf(struct dma *dma, void *dst,
505 const struct bcm6348_iudma_priv *priv = dev_get_priv(dma->dev);
506 struct bcm6348_chan_priv *ch_priv = priv->ch_priv[dma->id];
508 /* only add new rx buffers if channel isn't running */
509 if (ch_priv->running)
510 return bcm6348_iudma_free_rcv_buf(dma, dst, size);
512 return bcm6348_iudma_add_rcv_buf(dma, dst, size);
515 static const struct dma_ops bcm6348_iudma_ops = {
516 .disable = bcm6348_iudma_disable,
517 .enable = bcm6348_iudma_enable,
518 .prepare_rcv_buf = bcm6348_iudma_prepare_rcv_buf,
519 .request = bcm6348_iudma_request,
520 .receive = bcm6348_iudma_receive,
521 .send = bcm6348_iudma_send,
524 static const struct bcm6348_iudma_hw bcm6348_hw = {
525 .err_mask = (DMAD6348_ST_OV_ERR_MASK |
526 DMAD6348_ST_CRC_ERR_MASK |
527 DMAD6348_ST_RX_ERR_MASK |
528 DMAD6348_ST_OS_ERR_MASK |
529 DMAD6348_ST_UN_ERR_MASK),
532 static const struct bcm6348_iudma_hw bcm6368_hw = {
536 static const struct udevice_id bcm6348_iudma_ids[] = {
538 .compatible = "brcm,bcm6348-iudma",
539 .data = (ulong)&bcm6348_hw,
541 .compatible = "brcm,bcm6368-iudma",
542 .data = (ulong)&bcm6368_hw,
543 }, { /* sentinel */ }
546 static int bcm6348_iudma_probe(struct udevice *dev)
548 struct dma_dev_priv *uc_priv = dev_get_uclass_priv(dev);
549 struct bcm6348_iudma_priv *priv = dev_get_priv(dev);
550 const struct bcm6348_iudma_hw *hw =
551 (const struct bcm6348_iudma_hw *)dev_get_driver_data(dev);
555 uc_priv->supported = (DMA_SUPPORTS_DEV_TO_MEM |
556 DMA_SUPPORTS_MEM_TO_DEV);
559 /* dma global base address */
560 priv->base = dev_remap_addr_name(dev, "dma");
564 /* dma channels base address */
565 priv->chan = dev_remap_addr_name(dev, "dma-channels");
569 /* dma sram base address */
570 priv->sram = dev_remap_addr_name(dev, "dma-sram");
574 /* get number of channels */
575 priv->n_channels = dev_read_u32_default(dev, "dma-channels", 8);
576 if (priv->n_channels > DMA_CHAN_MAX)
579 /* try to enable clocks */
584 ret = clk_get_by_index(dev, i, &clk);
588 ret = clk_enable(&clk);
590 pr_err("error enabling clock %d\n", i);
594 ret = clk_free(&clk);
596 pr_err("error freeing clock %d\n", i);
601 /* try to perform resets */
603 struct reset_ctl reset;
606 ret = reset_get_by_index(dev, i, &reset);
610 ret = reset_deassert(&reset);
612 pr_err("error deasserting reset %d\n", i);
616 ret = reset_free(&reset);
618 pr_err("error freeing reset %d\n", i);
623 /* disable dma controller */
624 clrbits_be32(priv->base + DMA_CFG_REG, DMA_CFG_ENABLE_MASK);
626 /* alloc channel private data pointers */
627 priv->ch_priv = calloc(priv->n_channels,
628 sizeof(struct bcm6348_chan_priv*));
632 /* stop dma channels */
633 for (ch = 0; ch < priv->n_channels; ch++)
634 bcm6348_iudma_chan_stop(priv, ch);
636 /* enable dma controller */
637 setbits_be32(priv->base + DMA_CFG_REG, DMA_CFG_ENABLE_MASK);
642 U_BOOT_DRIVER(bcm6348_iudma) = {
643 .name = "bcm6348_iudma",
645 .of_match = bcm6348_iudma_ids,
646 .ops = &bcm6348_iudma_ops,
647 .priv_auto_alloc_size = sizeof(struct bcm6348_iudma_priv),
648 .probe = bcm6348_iudma_probe,