1 // SPDX-License-Identifier: GPL-2.0+
3 * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
5 * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
11 #include <asm/dma-mapping.h>
12 #include <asm/bitops.h>
15 #include <dm/uclass.h>
16 #include <linux/compat.h>
17 #include <linux/soc/ti/k3-navss-ringacc.h>
18 #include <linux/soc/ti/ti_sci_protocol.h>
20 #define set_bit(bit, bitmap) __set_bit(bit, bitmap)
21 #define clear_bit(bit, bitmap) __clear_bit(bit, bitmap)
22 #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
23 dma_free_coherent(cpu_addr)
24 #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
26 void *ring_mem_virt; \
27 ring_mem_virt = dma_alloc_coherent((size), \
28 (unsigned long *)(dma_handle)); \
30 memset(ring_mem_virt, 0, (size)); \
34 static LIST_HEAD(k3_nav_ringacc_list);
36 static void ringacc_writel(u32 v, void __iomem *reg)
38 pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
42 static u32 ringacc_readl(void __iomem *reg)
47 pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
51 #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
54 * struct k3_nav_ring_rt_regs - The RA Control/Status Registers region
56 struct k3_nav_ring_rt_regs {
58 u32 db; /* RT Ring N Doorbell Register */
60 u32 occ; /* RT Ring N Occupancy Register */
61 u32 indx; /* RT Ring N Current Index Register */
62 u32 hwocc; /* RT Ring N Hardware Occupancy Register */
63 u32 hwindx; /* RT Ring N Current Index Register */
66 #define KNAV_RINGACC_RT_REGS_STEP 0x1000
69 * struct k3_nav_ring_fifo_regs - The Ring Accelerator Queues Registers region
71 struct k3_nav_ring_fifo_regs {
72 u32 head_data[128]; /* Ring Head Entry Data Registers */
73 u32 tail_data[128]; /* Ring Tail Entry Data Registers */
74 u32 peek_head_data[128]; /* Ring Peek Head Entry Data Regs */
75 u32 peek_tail_data[128]; /* Ring Peek Tail Entry Data Regs */
79 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
81 struct k3_ringacc_proxy_gcfg_regs {
82 u32 revision; /* Revision Register */
83 u32 config; /* Config Register */
86 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0)
89 * struct k3_ringacc_proxy_target_regs - RA Proxy Datapath MMIO Region
91 struct k3_ringacc_proxy_target_regs {
92 u32 control; /* Proxy Control Register */
93 u32 status; /* Proxy Status Register */
95 u32 data[128]; /* Proxy Data Register */
98 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000
99 #define K3_RINGACC_PROXY_NOT_USED (-1)
101 enum k3_ringacc_proxy_access_mode {
102 PROXY_ACCESS_MODE_HEAD = 0,
103 PROXY_ACCESS_MODE_TAIL = 1,
104 PROXY_ACCESS_MODE_PEEK_HEAD = 2,
105 PROXY_ACCESS_MODE_PEEK_TAIL = 3,
108 #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
109 #define KNAV_RINGACC_FIFO_REGS_STEP 0x1000
110 #define KNAV_RINGACC_MAX_DB_RING_CNT (127U)
113 * struct k3_nav_ring_ops - Ring operations
115 struct k3_nav_ring_ops {
116 int (*push_tail)(struct k3_nav_ring *ring, void *elm);
117 int (*push_head)(struct k3_nav_ring *ring, void *elm);
118 int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
119 int (*pop_head)(struct k3_nav_ring *ring, void *elm);
123 * struct k3_nav_ring - RA Ring descriptor
125 * @rt - Ring control/status registers
126 * @fifos - Ring queues registers
127 * @proxy - Ring Proxy Datapath registers
128 * @ring_mem_dma - Ring buffer dma address
129 * @ring_mem_virt - Ring buffer virt address
130 * @ops - Ring operations
131 * @size - Ring size in elements
132 * @elm_size - Size of the ring element
135 * @free - Number of free elements
136 * @occ - Ring occupancy
137 * @windex - Write index (only for @K3_NAV_RINGACC_RING_MODE_RING)
138 * @rindex - Read index (only for @K3_NAV_RINGACC_RING_MODE_RING)
140 * @parent - Pointer on struct @k3_nav_ringacc
141 * @use_count - Use count for shared rings
142 * @proxy_id - RA Ring Proxy Id (only if @K3_NAV_RINGACC_RING_USE_PROXY)
145 struct k3_nav_ring_rt_regs __iomem *rt;
146 struct k3_nav_ring_fifo_regs __iomem *fifos;
147 struct k3_ringacc_proxy_target_regs __iomem *proxy;
148 dma_addr_t ring_mem_dma;
150 struct k3_nav_ring_ops *ops;
152 enum k3_nav_ring_size elm_size;
153 enum k3_nav_ring_mode mode;
155 #define KNAV_RING_FLAG_BUSY BIT(1)
156 #define K3_NAV_RING_FLAG_SHARED BIT(2)
162 struct k3_nav_ringacc *parent;
168 * struct k3_nav_ringacc - Rings accelerator descriptor
170 * @dev - pointer on RA device
171 * @proxy_gcfg - RA proxy global config registers
172 * @proxy_target_base - RA proxy datapath region
173 * @num_rings - number of ring in RA
174 * @rm_gp_range - general purpose rings range from tisci
175 * @dma_ring_reset_quirk - DMA reset w/a enable
176 * @num_proxies - number of RA proxies
177 * @rings - array of rings descriptors (struct @k3_nav_ring)
178 * @list - list of RAs in the system
179 * @tisci - pointer ti-sci handle
180 * @tisci_ring_ops - ti-sci rings ops
181 * @tisci_dev_id - ti-sci device id
183 struct k3_nav_ringacc {
185 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
186 void __iomem *proxy_target_base;
187 u32 num_rings; /* number of rings in Ringacc module */
188 unsigned long *rings_inuse;
189 struct ti_sci_resource *rm_gp_range;
190 bool dma_ring_reset_quirk;
192 unsigned long *proxy_inuse;
194 struct k3_nav_ring *rings;
195 struct list_head list;
197 const struct ti_sci_handle *tisci;
198 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
202 static long k3_nav_ringacc_ring_get_fifo_pos(struct k3_nav_ring *ring)
204 return KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES -
205 (4 << ring->elm_size);
208 static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
210 return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
213 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
214 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
216 static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
217 .push_tail = k3_nav_ringacc_ring_push_mem,
218 .pop_head = k3_nav_ringacc_ring_pop_mem,
221 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem);
222 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem);
223 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
225 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring,
228 static struct k3_nav_ring_ops k3_nav_mode_msg_ops = {
229 .push_tail = k3_nav_ringacc_ring_push_io,
230 .push_head = k3_nav_ringacc_ring_push_head_io,
231 .pop_tail = k3_nav_ringacc_ring_pop_tail_io,
232 .pop_head = k3_nav_ringacc_ring_pop_io,
235 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring,
237 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring,
239 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem);
240 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem);
242 static struct k3_nav_ring_ops k3_nav_mode_proxy_ops = {
243 .push_tail = k3_ringacc_ring_push_tail_proxy,
244 .push_head = k3_ringacc_ring_push_head_proxy,
245 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
246 .pop_head = k3_ringacc_ring_pop_head_proxy,
249 struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
254 struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
257 int proxy_id = K3_RINGACC_PROXY_NOT_USED;
259 if (id == K3_NAV_RINGACC_RING_ID_ANY) {
260 /* Request for any general purpose ring */
261 struct ti_sci_resource_desc *gp_rings =
262 &ringacc->rm_gp_range->desc[0];
265 size = gp_rings->start + gp_rings->num;
266 id = find_next_zero_bit(ringacc->rings_inuse,
267 size, gp_rings->start);
274 if (test_bit(id, ringacc->rings_inuse) &&
275 !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
277 else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
280 if (flags & K3_NAV_RINGACC_RING_USE_PROXY) {
281 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
282 ringacc->num_proxies, 0);
283 if (proxy_id == ringacc->num_proxies)
287 if (!try_module_get(ringacc->dev->driver->owner))
290 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
291 set_bit(proxy_id, ringacc->proxy_inuse);
292 ringacc->rings[id].proxy_id = proxy_id;
293 pr_debug("Giving ring#%d proxy#%d\n",
296 pr_debug("Giving ring#%d\n", id);
299 set_bit(id, ringacc->rings_inuse);
301 ringacc->rings[id].use_count++;
302 return &ringacc->rings[id];
308 static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
310 struct k3_nav_ringacc *ringacc = ring->parent;
313 ret = ringacc->tisci_ring_ops->config(
315 TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
316 ringacc->tisci_dev_id,
325 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
329 void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
331 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
339 k3_ringacc_ring_reset_sci(ring);
342 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
343 enum k3_nav_ring_mode mode)
345 struct k3_nav_ringacc *ringacc = ring->parent;
348 ret = ringacc->tisci_ring_ops->config(
350 TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
351 ringacc->tisci_dev_id,
360 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
364 void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
366 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
369 if (!ring->parent->dma_ring_reset_quirk) {
370 k3_nav_ringacc_ring_reset(ring);
375 occ = ringacc_readl(&ring->rt->occ);
378 u32 db_ring_cnt, db_ring_cnt_cur;
380 pr_debug("%s %u occ: %u\n", __func__,
382 /* 2. Reset the ring */
383 k3_ringacc_ring_reset_sci(ring);
386 * 3. Setup the ring in ring/doorbell mode
387 * (if not already in this mode)
389 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
390 k3_ringacc_ring_reconfig_qmode_sci(
391 ring, K3_NAV_RINGACC_RING_MODE_RING);
393 * 4. Ring the doorbell 2**22 – ringOcc times.
394 * This will wrap the internal UDMAP ring state occupancy
395 * counter (which is 21-bits wide) to 0.
397 db_ring_cnt = (1U << 22) - occ;
399 while (db_ring_cnt != 0) {
401 * Ring the doorbell with the maximum count each
402 * iteration if possible to minimize the total
405 if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
406 db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
408 db_ring_cnt_cur = db_ring_cnt;
410 writel(db_ring_cnt_cur, &ring->rt->db);
411 db_ring_cnt -= db_ring_cnt_cur;
414 /* 5. Restore the original ring mode (if not ring mode) */
415 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
416 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
419 /* 2. Reset the ring */
420 k3_nav_ringacc_ring_reset(ring);
423 static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
425 struct k3_nav_ringacc *ringacc = ring->parent;
428 ret = ringacc->tisci_ring_ops->config(
430 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
431 ringacc->tisci_dev_id,
440 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
444 int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
446 struct k3_nav_ringacc *ringacc;
451 ringacc = ring->parent;
453 pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
455 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
458 if (--ring->use_count)
461 if (!(ring->flags & KNAV_RING_FLAG_BUSY))
464 k3_ringacc_ring_free_sci(ring);
466 dma_free_coherent(ringacc->dev,
467 ring->size * (4 << ring->elm_size),
468 ring->ring_mem_virt, ring->ring_mem_dma);
469 ring->flags &= ~KNAV_RING_FLAG_BUSY;
471 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
472 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
474 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
478 clear_bit(ring->ring_id, ringacc->rings_inuse);
480 module_put(ringacc->dev->driver->owner);
486 u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
491 return ring->ring_id;
494 static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
496 struct k3_nav_ringacc *ringacc = ring->parent;
503 ring_idx = ring->ring_id;
504 ret = ringacc->tisci_ring_ops->config(
506 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
507 ringacc->tisci_dev_id,
509 lower_32_bits(ring->ring_mem_dma),
510 upper_32_bits(ring->ring_mem_dma),
516 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
522 int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
523 struct k3_nav_ring_cfg *cfg)
525 struct k3_nav_ringacc *ringacc = ring->parent;
530 if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
531 cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
532 cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
533 !test_bit(ring->ring_id, ringacc->rings_inuse))
536 if (ring->use_count != 1)
539 ring->size = cfg->size;
540 ring->elm_size = cfg->elm_size;
541 ring->mode = cfg->mode;
547 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
548 ring->proxy = ringacc->proxy_target_base +
549 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
551 switch (ring->mode) {
552 case K3_NAV_RINGACC_RING_MODE_RING:
553 ring->ops = &k3_nav_mode_ring_ops;
555 case K3_NAV_RINGACC_RING_MODE_QM:
557 * In Queue mode elm_size can be 8 only and each operation
558 * uses 2 element slots
560 if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
563 case K3_NAV_RINGACC_RING_MODE_MESSAGE:
565 ring->ops = &k3_nav_mode_proxy_ops;
567 ring->ops = &k3_nav_mode_msg_ops;
575 ring->ring_mem_virt =
576 dma_zalloc_coherent(ringacc->dev,
577 ring->size * (4 << ring->elm_size),
578 &ring->ring_mem_dma, GFP_KERNEL);
579 if (!ring->ring_mem_virt) {
580 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
585 ret = k3_nav_ringacc_ring_cfg_sci(ring);
590 ring->flags |= KNAV_RING_FLAG_BUSY;
591 ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
592 K3_NAV_RING_FLAG_SHARED : 0;
597 dma_free_coherent(ringacc->dev,
598 ring->size * (4 << ring->elm_size),
608 u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
610 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
616 u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
618 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
622 ring->free = ring->size - ringacc_readl(&ring->rt->occ);
627 u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
629 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
632 return ringacc_readl(&ring->rt->occ);
635 u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
637 return !k3_nav_ringacc_ring_get_free(ring);
640 enum k3_ringacc_access_mode {
641 K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
642 K3_RINGACC_ACCESS_MODE_POP_HEAD,
643 K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
644 K3_RINGACC_ACCESS_MODE_POP_TAIL,
645 K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
646 K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
649 static int k3_ringacc_ring_cfg_proxy(struct k3_nav_ring *ring,
650 enum k3_ringacc_proxy_access_mode mode)
656 val |= ring->elm_size << 24;
657 ringacc_writel(val, &ring->proxy->control);
661 static int k3_nav_ringacc_ring_access_proxy(
662 struct k3_nav_ring *ring, void *elem,
663 enum k3_ringacc_access_mode access_mode)
667 ptr = (void __iomem *)&ring->proxy->data;
669 switch (access_mode) {
670 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
671 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
672 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
674 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
675 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
676 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
682 ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
684 switch (access_mode) {
685 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
686 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
687 pr_debug("proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
689 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
692 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
693 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
694 pr_debug("proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
696 memcpy_toio(ptr, elem, (4 << ring->elm_size));
703 pr_debug("proxy: free%d occ%d\n",
704 ring->free, ring->occ);
708 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring, void *elem)
710 return k3_nav_ringacc_ring_access_proxy(
711 ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
714 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring, void *elem)
716 return k3_nav_ringacc_ring_access_proxy(
717 ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
720 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem)
722 return k3_nav_ringacc_ring_access_proxy(
723 ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
726 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem)
728 return k3_nav_ringacc_ring_access_proxy(
729 ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
732 static int k3_nav_ringacc_ring_access_io(
733 struct k3_nav_ring *ring, void *elem,
734 enum k3_ringacc_access_mode access_mode)
738 switch (access_mode) {
739 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
740 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
741 ptr = (void __iomem *)&ring->fifos->head_data;
743 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
744 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
745 ptr = (void __iomem *)&ring->fifos->tail_data;
751 ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
753 switch (access_mode) {
754 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
755 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
756 pr_debug("memcpy_fromio(x): --> ptr(%p), mode:%d\n",
758 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
761 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
762 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
763 pr_debug("memcpy_toio(x): --> ptr(%p), mode:%d\n",
765 memcpy_toio(ptr, elem, (4 << ring->elm_size));
772 pr_debug("free%d index%d occ%d index%d\n",
773 ring->free, ring->windex, ring->occ, ring->rindex);
777 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
780 return k3_nav_ringacc_ring_access_io(
781 ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
784 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem)
786 return k3_nav_ringacc_ring_access_io(
787 ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
790 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem)
792 return k3_nav_ringacc_ring_access_io(
793 ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
796 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring, void *elem)
798 return k3_nav_ringacc_ring_access_io(
799 ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
802 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
806 elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->windex);
808 memcpy(elem_ptr, elem, (4 << ring->elm_size));
810 ring->windex = (ring->windex + 1) % ring->size;
812 ringacc_writel(1, &ring->rt->db);
814 pr_debug("ring_push_mem: free%d index%d\n",
815 ring->free, ring->windex);
820 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
824 elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->rindex);
826 memcpy(elem, elem_ptr, (4 << ring->elm_size));
828 ring->rindex = (ring->rindex + 1) % ring->size;
830 ringacc_writel(-1, &ring->rt->db);
832 pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
833 ring->occ, ring->rindex, elem_ptr);
837 int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
839 int ret = -EOPNOTSUPP;
841 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
844 pr_debug("ring_push%d: free%d index%d\n",
845 ring->ring_id, ring->free, ring->windex);
847 if (k3_nav_ringacc_ring_is_full(ring))
850 if (ring->ops && ring->ops->push_tail)
851 ret = ring->ops->push_tail(ring, elem);
856 int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
858 int ret = -EOPNOTSUPP;
860 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
863 pr_debug("ring_push_head: free%d index%d\n",
864 ring->free, ring->windex);
866 if (k3_nav_ringacc_ring_is_full(ring))
869 if (ring->ops && ring->ops->push_head)
870 ret = ring->ops->push_head(ring, elem);
875 int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
877 int ret = -EOPNOTSUPP;
879 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
883 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
885 pr_debug("ring_pop%d: occ%d index%d\n",
886 ring->ring_id, ring->occ, ring->rindex);
891 if (ring->ops && ring->ops->pop_head)
892 ret = ring->ops->pop_head(ring, elem);
897 int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
899 int ret = -EOPNOTSUPP;
901 if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
905 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
907 pr_debug("ring_pop_tail: occ%d index%d\n",
908 ring->occ, ring->rindex);
913 if (ring->ops && ring->ops->pop_tail)
914 ret = ring->ops->pop_tail(ring, elem);
919 static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
921 struct udevice *dev = ringacc->dev;
922 struct udevice *tisci_dev = NULL;
925 ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
926 if (!ringacc->num_rings) {
927 dev_err(dev, "ti,num-rings read failure %d\n", ret);
931 ringacc->dma_ring_reset_quirk =
932 dev_read_bool(dev, "ti,dma-ring-reset-quirk");
934 ret = uclass_get_device_by_name(UCLASS_FIRMWARE, "dmsc", &tisci_dev);
936 pr_debug("TISCI RA RM get failed (%d)\n", ret);
937 ringacc->tisci = NULL;
940 ringacc->tisci = (struct ti_sci_handle *)
941 (ti_sci_get_handle_from_sysfw(tisci_dev));
943 ret = dev_read_u32_default(dev, "ti,sci", 0);
945 dev_err(dev, "TISCI RA RM disabled\n");
946 ringacc->tisci = NULL;
950 ret = dev_read_u32(dev, "ti,sci-dev-id", &ringacc->tisci_dev_id);
952 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
953 ringacc->tisci = NULL;
957 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
959 ringacc->tisci_dev_id,
960 "ti,sci-rm-range-gp-rings");
961 if (IS_ERR(ringacc->rm_gp_range))
962 ret = PTR_ERR(ringacc->rm_gp_range);
967 static int k3_nav_ringacc_probe(struct udevice *dev)
969 struct k3_nav_ringacc *ringacc;
970 void __iomem *base_fifo, *base_rt;
973 ringacc = dev_get_priv(dev);
979 ret = k3_nav_ringacc_probe_dt(ringacc);
983 base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
984 pr_debug("rt %p\n", base_rt);
986 return PTR_ERR(base_rt);
988 base_fifo = (uint32_t *)devfdt_get_addr_name(dev, "fifos");
989 pr_debug("fifos %p\n", base_fifo);
990 if (IS_ERR(base_fifo))
991 return PTR_ERR(base_fifo);
993 ringacc->proxy_gcfg = (struct k3_ringacc_proxy_gcfg_regs __iomem *)
994 devfdt_get_addr_name(dev, "proxy_gcfg");
995 if (IS_ERR(ringacc->proxy_gcfg))
996 return PTR_ERR(ringacc->proxy_gcfg);
997 ringacc->proxy_target_base =
998 (struct k3_ringacc_proxy_gcfg_regs __iomem *)
999 devfdt_get_addr_name(dev, "proxy_target");
1000 if (IS_ERR(ringacc->proxy_target_base))
1001 return PTR_ERR(ringacc->proxy_target_base);
1003 ringacc->num_proxies = ringacc_readl(&ringacc->proxy_gcfg->config) &
1004 K3_RINGACC_PROXY_CFG_THREADS_MASK;
1006 ringacc->rings = devm_kzalloc(dev,
1007 sizeof(*ringacc->rings) *
1010 ringacc->rings_inuse = devm_kcalloc(dev,
1011 BITS_TO_LONGS(ringacc->num_rings),
1012 sizeof(unsigned long), GFP_KERNEL);
1013 ringacc->proxy_inuse = devm_kcalloc(dev,
1014 BITS_TO_LONGS(ringacc->num_proxies),
1015 sizeof(unsigned long), GFP_KERNEL);
1017 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1020 for (i = 0; i < ringacc->num_rings; i++) {
1021 ringacc->rings[i].rt = base_rt +
1022 KNAV_RINGACC_RT_REGS_STEP * i;
1023 ringacc->rings[i].fifos = base_fifo +
1024 KNAV_RINGACC_FIFO_REGS_STEP * i;
1025 ringacc->rings[i].parent = ringacc;
1026 ringacc->rings[i].ring_id = i;
1027 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1029 dev_set_drvdata(dev, ringacc);
1031 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1033 list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
1035 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1037 ringacc->rm_gp_range->desc[0].start,
1038 ringacc->rm_gp_range->desc[0].num,
1039 ringacc->tisci_dev_id);
1040 dev_info(dev, "dma-ring-reset-quirk: %s\n",
1041 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1042 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1043 ringacc_readl(&ringacc->proxy_gcfg->revision),
1044 ringacc->num_proxies);
1048 static const struct udevice_id knav_ringacc_ids[] = {
1049 { .compatible = "ti,am654-navss-ringacc" },
1053 U_BOOT_DRIVER(k3_navss_ringacc) = {
1054 .name = "k3-navss-ringacc",
1056 .of_match = knav_ringacc_ids,
1057 .probe = k3_nav_ringacc_probe,
1058 .priv_auto_alloc_size = sizeof(struct k3_nav_ringacc),