--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#include <common.h>
+#include <asm/io.h>
+#include <malloc.h>
+#include <asm/dma-mapping.h>
+#include <asm/bitops.h>
+#include <dm.h>
+#include <dm/read.h>
+#include <dm/uclass.h>
+#include <linux/compat.h>
+#include <linux/soc/ti/k3-navss-ringacc.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+
+#define set_bit(bit, bitmap) __set_bit(bit, bitmap)
+#define clear_bit(bit, bitmap) __clear_bit(bit, bitmap)
+#define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
+ dma_free_coherent(cpu_addr)
+#define dma_zalloc_coherent(dev, size, dma_handle, flag) \
+({ \
+ void *ring_mem_virt; \
+ ring_mem_virt = dma_alloc_coherent((size), \
+ (unsigned long *)(dma_handle)); \
+ if (ring_mem_virt) \
+ memset(ring_mem_virt, 0, (size)); \
+ ring_mem_virt; \
+})
+
+static LIST_HEAD(k3_nav_ringacc_list);
+
+static void ringacc_writel(u32 v, void __iomem *reg)
+{
+ pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
+ writel(v, reg);
+}
+
+static u32 ringacc_readl(void __iomem *reg)
+{
+ u32 v;
+
+ v = readl(reg);
+ pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
+ return v;
+}
+
+#define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
+
+/**
+ * struct k3_nav_ring_rt_regs - The RA Control/Status Registers region
+ */
+struct k3_nav_ring_rt_regs {
+ u32 resv_16[4];
+ u32 db; /* RT Ring N Doorbell Register */
+ u32 resv_4[1];
+ u32 occ; /* RT Ring N Occupancy Register */
+ u32 indx; /* RT Ring N Current Index Register */
+ u32 hwocc; /* RT Ring N Hardware Occupancy Register */
+ u32 hwindx; /* RT Ring N Current Index Register */
+};
+
+#define KNAV_RINGACC_RT_REGS_STEP 0x1000
+
+/**
+ * struct k3_nav_ring_fifo_regs - The Ring Accelerator Queues Registers region
+ */
+struct k3_nav_ring_fifo_regs {
+ u32 head_data[128]; /* Ring Head Entry Data Registers */
+ u32 tail_data[128]; /* Ring Tail Entry Data Registers */
+ u32 peek_head_data[128]; /* Ring Peek Head Entry Data Regs */
+ u32 peek_tail_data[128]; /* Ring Peek Tail Entry Data Regs */
+};
+
+/**
+ * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
+ */
+struct k3_ringacc_proxy_gcfg_regs {
+ u32 revision; /* Revision Register */
+ u32 config; /* Config Register */
+};
+
+#define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0)
+
+/**
+ * struct k3_ringacc_proxy_target_regs - RA Proxy Datapath MMIO Region
+ */
+struct k3_ringacc_proxy_target_regs {
+ u32 control; /* Proxy Control Register */
+ u32 status; /* Proxy Status Register */
+ u8 resv_512[504];
+ u32 data[128]; /* Proxy Data Register */
+};
+
+#define K3_RINGACC_PROXY_TARGET_STEP 0x1000
+#define K3_RINGACC_PROXY_NOT_USED (-1)
+
+enum k3_ringacc_proxy_access_mode {
+ PROXY_ACCESS_MODE_HEAD = 0,
+ PROXY_ACCESS_MODE_TAIL = 1,
+ PROXY_ACCESS_MODE_PEEK_HEAD = 2,
+ PROXY_ACCESS_MODE_PEEK_TAIL = 3,
+};
+
+#define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
+#define KNAV_RINGACC_FIFO_REGS_STEP 0x1000
+#define KNAV_RINGACC_MAX_DB_RING_CNT (127U)
+
+/**
+ * struct k3_nav_ring_ops - Ring operations
+ */
+struct k3_nav_ring_ops {
+ int (*push_tail)(struct k3_nav_ring *ring, void *elm);
+ int (*push_head)(struct k3_nav_ring *ring, void *elm);
+ int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
+ int (*pop_head)(struct k3_nav_ring *ring, void *elm);
+};
+
+/**
+ * struct k3_nav_ring - RA Ring descriptor
+ *
+ * @rt - Ring control/status registers
+ * @fifos - Ring queues registers
+ * @proxy - Ring Proxy Datapath registers
+ * @ring_mem_dma - Ring buffer dma address
+ * @ring_mem_virt - Ring buffer virt address
+ * @ops - Ring operations
+ * @size - Ring size in elements
+ * @elm_size - Size of the ring element
+ * @mode - Ring mode
+ * @flags - flags
+ * @free - Number of free elements
+ * @occ - Ring occupancy
+ * @windex - Write index (only for @K3_NAV_RINGACC_RING_MODE_RING)
+ * @rindex - Read index (only for @K3_NAV_RINGACC_RING_MODE_RING)
+ * @ring_id - Ring Id
+ * @parent - Pointer on struct @k3_nav_ringacc
+ * @use_count - Use count for shared rings
+ * @proxy_id - RA Ring Proxy Id (only if @K3_NAV_RINGACC_RING_USE_PROXY)
+ */
+struct k3_nav_ring {
+ struct k3_nav_ring_rt_regs __iomem *rt;
+ struct k3_nav_ring_fifo_regs __iomem *fifos;
+ struct k3_ringacc_proxy_target_regs __iomem *proxy;
+ dma_addr_t ring_mem_dma;
+ void *ring_mem_virt;
+ struct k3_nav_ring_ops *ops;
+ u32 size;
+ enum k3_nav_ring_size elm_size;
+ enum k3_nav_ring_mode mode;
+ u32 flags;
+#define KNAV_RING_FLAG_BUSY BIT(1)
+#define K3_NAV_RING_FLAG_SHARED BIT(2)
+ u32 free;
+ u32 occ;
+ u32 windex;
+ u32 rindex;
+ u32 ring_id;
+ struct k3_nav_ringacc *parent;
+ u32 use_count;
+ int proxy_id;
+};
+
+/**
+ * struct k3_nav_ringacc - Rings accelerator descriptor
+ *
+ * @dev - pointer on RA device
+ * @proxy_gcfg - RA proxy global config registers
+ * @proxy_target_base - RA proxy datapath region
+ * @num_rings - number of ring in RA
+ * @rm_gp_range - general purpose rings range from tisci
+ * @dma_ring_reset_quirk - DMA reset w/a enable
+ * @num_proxies - number of RA proxies
+ * @rings - array of rings descriptors (struct @k3_nav_ring)
+ * @list - list of RAs in the system
+ * @tisci - pointer ti-sci handle
+ * @tisci_ring_ops - ti-sci rings ops
+ * @tisci_dev_id - ti-sci device id
+ */
+struct k3_nav_ringacc {
+ struct udevice *dev;
+ struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
+ void __iomem *proxy_target_base;
+ u32 num_rings; /* number of rings in Ringacc module */
+ unsigned long *rings_inuse;
+ struct ti_sci_resource *rm_gp_range;
+ bool dma_ring_reset_quirk;
+ u32 num_proxies;
+ unsigned long *proxy_inuse;
+
+ struct k3_nav_ring *rings;
+ struct list_head list;
+
+ const struct ti_sci_handle *tisci;
+ const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
+ u32 tisci_dev_id;
+};
+
+static long k3_nav_ringacc_ring_get_fifo_pos(struct k3_nav_ring *ring)
+{
+ return KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES -
+ (4 << ring->elm_size);
+}
+
+static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
+{
+ return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
+}
+
+static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
+static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
+
+static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
+ .push_tail = k3_nav_ringacc_ring_push_mem,
+ .pop_head = k3_nav_ringacc_ring_pop_mem,
+};
+
+static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem);
+static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem);
+static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
+ void *elem);
+static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring,
+ void *elem);
+
+static struct k3_nav_ring_ops k3_nav_mode_msg_ops = {
+ .push_tail = k3_nav_ringacc_ring_push_io,
+ .push_head = k3_nav_ringacc_ring_push_head_io,
+ .pop_tail = k3_nav_ringacc_ring_pop_tail_io,
+ .pop_head = k3_nav_ringacc_ring_pop_io,
+};
+
+static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring,
+ void *elem);
+static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring,
+ void *elem);
+static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem);
+static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem);
+
+static struct k3_nav_ring_ops k3_nav_mode_proxy_ops = {
+ .push_tail = k3_ringacc_ring_push_tail_proxy,
+ .push_head = k3_ringacc_ring_push_head_proxy,
+ .pop_tail = k3_ringacc_ring_pop_tail_proxy,
+ .pop_head = k3_ringacc_ring_pop_head_proxy,
+};
+
+struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
+{
+ return ringacc->dev;
+}
+
+struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
+ int id, u32 flags)
+{
+ int proxy_id = K3_RINGACC_PROXY_NOT_USED;
+
+ if (id == K3_NAV_RINGACC_RING_ID_ANY) {
+ /* Request for any general purpose ring */
+ struct ti_sci_resource_desc *gp_rings =
+ &ringacc->rm_gp_range->desc[0];
+ unsigned long size;
+
+ size = gp_rings->start + gp_rings->num;
+ id = find_next_zero_bit(ringacc->rings_inuse,
+ size, gp_rings->start);
+ if (id == size)
+ goto error;
+ } else if (id < 0) {
+ goto error;
+ }
+
+ if (test_bit(id, ringacc->rings_inuse) &&
+ !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
+ goto error;
+ else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
+ goto out;
+
+ if (flags & K3_NAV_RINGACC_RING_USE_PROXY) {
+ proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
+ ringacc->num_proxies, 0);
+ if (proxy_id == ringacc->num_proxies)
+ goto error;
+ }
+
+ if (!try_module_get(ringacc->dev->driver->owner))
+ goto error;
+
+ if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
+ set_bit(proxy_id, ringacc->proxy_inuse);
+ ringacc->rings[id].proxy_id = proxy_id;
+ pr_debug("Giving ring#%d proxy#%d\n",
+ id, proxy_id);
+ } else {
+ pr_debug("Giving ring#%d\n", id);
+ }
+
+ set_bit(id, ringacc->rings_inuse);
+out:
+ ringacc->rings[id].use_count++;
+ return &ringacc->rings[id];
+
+error:
+ return NULL;
+}
+
+static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
+{
+ struct k3_nav_ringacc *ringacc = ring->parent;
+ int ret;
+
+ ret = ringacc->tisci_ring_ops->config(
+ ringacc->tisci,
+ TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
+ ringacc->tisci_dev_id,
+ ring->ring_id,
+ 0,
+ 0,
+ ring->size,
+ 0,
+ 0,
+ 0);
+ if (ret)
+ dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
+ ret, ring->ring_id);
+}
+
+void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
+{
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return;
+
+ ring->occ = 0;
+ ring->free = 0;
+ ring->rindex = 0;
+ ring->windex = 0;
+
+ k3_ringacc_ring_reset_sci(ring);
+}
+
+static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
+ enum k3_nav_ring_mode mode)
+{
+ struct k3_nav_ringacc *ringacc = ring->parent;
+ int ret;
+
+ ret = ringacc->tisci_ring_ops->config(
+ ringacc->tisci,
+ TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
+ ringacc->tisci_dev_id,
+ ring->ring_id,
+ 0,
+ 0,
+ 0,
+ mode,
+ 0,
+ 0);
+ if (ret)
+ dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
+ ret, ring->ring_id);
+}
+
+void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
+{
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return;
+
+ if (!ring->parent->dma_ring_reset_quirk)
+ return;
+
+ if (!occ)
+ occ = ringacc_readl(&ring->rt->occ);
+
+ if (occ) {
+ u32 db_ring_cnt, db_ring_cnt_cur;
+
+ pr_debug("%s %u occ: %u\n", __func__,
+ ring->ring_id, occ);
+ /* 2. Reset the ring */
+ k3_ringacc_ring_reset_sci(ring);
+
+ /*
+ * 3. Setup the ring in ring/doorbell mode
+ * (if not already in this mode)
+ */
+ if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
+ k3_ringacc_ring_reconfig_qmode_sci(
+ ring, K3_NAV_RINGACC_RING_MODE_RING);
+ /*
+ * 4. Ring the doorbell 2**22 – ringOcc times.
+ * This will wrap the internal UDMAP ring state occupancy
+ * counter (which is 21-bits wide) to 0.
+ */
+ db_ring_cnt = (1U << 22) - occ;
+
+ while (db_ring_cnt != 0) {
+ /*
+ * Ring the doorbell with the maximum count each
+ * iteration if possible to minimize the total
+ * of writes
+ */
+ if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
+ db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
+ else
+ db_ring_cnt_cur = db_ring_cnt;
+
+ writel(db_ring_cnt_cur, &ring->rt->db);
+ db_ring_cnt -= db_ring_cnt_cur;
+ }
+
+ /* 5. Restore the original ring mode (if not ring mode) */
+ if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
+ k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
+ }
+
+ /* 2. Reset the ring */
+ k3_nav_ringacc_ring_reset(ring);
+}
+
+static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
+{
+ struct k3_nav_ringacc *ringacc = ring->parent;
+ int ret;
+
+ ret = ringacc->tisci_ring_ops->config(
+ ringacc->tisci,
+ TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
+ ringacc->tisci_dev_id,
+ ring->ring_id,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0);
+ if (ret)
+ dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
+ ret, ring->ring_id);
+}
+
+int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
+{
+ struct k3_nav_ringacc *ringacc;
+
+ if (!ring)
+ return -EINVAL;
+
+ ringacc = ring->parent;
+
+ pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
+
+ if (!test_bit(ring->ring_id, ringacc->rings_inuse))
+ return -EINVAL;
+
+ if (--ring->use_count)
+ goto out;
+
+ if (!(ring->flags & KNAV_RING_FLAG_BUSY))
+ goto no_init;
+
+ k3_ringacc_ring_free_sci(ring);
+
+ dma_free_coherent(ringacc->dev,
+ ring->size * (4 << ring->elm_size),
+ ring->ring_mem_virt, ring->ring_mem_dma);
+ ring->flags &= ~KNAV_RING_FLAG_BUSY;
+ ring->ops = NULL;
+ if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
+ clear_bit(ring->proxy_id, ringacc->proxy_inuse);
+ ring->proxy = NULL;
+ ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
+ }
+
+no_init:
+ clear_bit(ring->ring_id, ringacc->rings_inuse);
+
+ module_put(ringacc->dev->driver->owner);
+
+out:
+ return 0;
+}
+
+u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
+{
+ if (!ring)
+ return -EINVAL;
+
+ return ring->ring_id;
+}
+
+static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
+{
+ struct k3_nav_ringacc *ringacc = ring->parent;
+ u32 ring_idx;
+ int ret;
+
+ if (!ringacc->tisci)
+ return -EINVAL;
+
+ ring_idx = ring->ring_id;
+ ret = ringacc->tisci_ring_ops->config(
+ ringacc->tisci,
+ TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
+ ringacc->tisci_dev_id,
+ ring_idx,
+ lower_32_bits(ring->ring_mem_dma),
+ upper_32_bits(ring->ring_mem_dma),
+ ring->size,
+ ring->mode,
+ ring->elm_size,
+ 0);
+ if (ret)
+ dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
+ ret, ring_idx);
+
+ return ret;
+}
+
+int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
+ struct k3_nav_ring_cfg *cfg)
+{
+ struct k3_nav_ringacc *ringacc = ring->parent;
+ int ret = 0;
+
+ if (!ring || !cfg)
+ return -EINVAL;
+ if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
+ cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
+ cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
+ !test_bit(ring->ring_id, ringacc->rings_inuse))
+ return -EINVAL;
+
+ if (ring->use_count != 1)
+ return 0;
+
+ ring->size = cfg->size;
+ ring->elm_size = cfg->elm_size;
+ ring->mode = cfg->mode;
+ ring->occ = 0;
+ ring->free = 0;
+ ring->rindex = 0;
+ ring->windex = 0;
+
+ if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
+ ring->proxy = ringacc->proxy_target_base +
+ ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
+
+ switch (ring->mode) {
+ case K3_NAV_RINGACC_RING_MODE_RING:
+ ring->ops = &k3_nav_mode_ring_ops;
+ break;
+ case K3_NAV_RINGACC_RING_MODE_QM:
+ /*
+ * In Queue mode elm_size can be 8 only and each operation
+ * uses 2 element slots
+ */
+ if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
+ cfg->size % 2)
+ goto err_free_proxy;
+ case K3_NAV_RINGACC_RING_MODE_MESSAGE:
+ if (ring->proxy)
+ ring->ops = &k3_nav_mode_proxy_ops;
+ else
+ ring->ops = &k3_nav_mode_msg_ops;
+ break;
+ default:
+ ring->ops = NULL;
+ ret = -EINVAL;
+ goto err_free_proxy;
+ };
+
+ ring->ring_mem_virt =
+ dma_zalloc_coherent(ringacc->dev,
+ ring->size * (4 << ring->elm_size),
+ &ring->ring_mem_dma, GFP_KERNEL);
+ if (!ring->ring_mem_virt) {
+ dev_err(ringacc->dev, "Failed to alloc ring mem\n");
+ ret = -ENOMEM;
+ goto err_free_ops;
+ }
+
+ ret = k3_nav_ringacc_ring_cfg_sci(ring);
+
+ if (ret)
+ goto err_free_mem;
+
+ ring->flags |= KNAV_RING_FLAG_BUSY;
+ ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
+ K3_NAV_RING_FLAG_SHARED : 0;
+
+ return 0;
+
+err_free_mem:
+ dma_free_coherent(ringacc->dev,
+ ring->size * (4 << ring->elm_size),
+ ring->ring_mem_virt,
+ ring->ring_mem_dma);
+err_free_ops:
+ ring->ops = NULL;
+err_free_proxy:
+ ring->proxy = NULL;
+ return ret;
+}
+
+u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
+{
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return -EINVAL;
+
+ return ring->size;
+}
+
+u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
+{
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return -EINVAL;
+
+ if (!ring->free)
+ ring->free = ring->size - ringacc_readl(&ring->rt->occ);
+
+ return ring->free;
+}
+
+u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
+{
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return -EINVAL;
+
+ return ringacc_readl(&ring->rt->occ);
+}
+
+u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
+{
+ return !k3_nav_ringacc_ring_get_free(ring);
+}
+
+enum k3_ringacc_access_mode {
+ K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
+ K3_RINGACC_ACCESS_MODE_POP_HEAD,
+ K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
+ K3_RINGACC_ACCESS_MODE_POP_TAIL,
+ K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
+ K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
+};
+
+static int k3_ringacc_ring_cfg_proxy(struct k3_nav_ring *ring,
+ enum k3_ringacc_proxy_access_mode mode)
+{
+ u32 val;
+
+ val = ring->ring_id;
+ val |= mode << 16;
+ val |= ring->elm_size << 24;
+ ringacc_writel(val, &ring->proxy->control);
+ return 0;
+}
+
+static int k3_nav_ringacc_ring_access_proxy(
+ struct k3_nav_ring *ring, void *elem,
+ enum k3_ringacc_access_mode access_mode)
+{
+ void __iomem *ptr;
+
+ ptr = (void __iomem *)&ring->proxy->data;
+
+ switch (access_mode) {
+ case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+ case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+ k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
+ break;
+ case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+ case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+ k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
+
+ switch (access_mode) {
+ case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+ case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+ pr_debug("proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
+ ptr, access_mode);
+ memcpy_fromio(elem, ptr, (4 << ring->elm_size));
+ ring->occ--;
+ break;
+ case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+ case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+ pr_debug("proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
+ ptr, access_mode);
+ memcpy_toio(ptr, elem, (4 << ring->elm_size));
+ ring->free--;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ pr_debug("proxy: free%d occ%d\n",
+ ring->free, ring->occ);
+ return 0;
+}
+
+static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring, void *elem)
+{
+ return k3_nav_ringacc_ring_access_proxy(
+ ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
+}
+
+static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring, void *elem)
+{
+ return k3_nav_ringacc_ring_access_proxy(
+ ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
+}
+
+static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem)
+{
+ return k3_nav_ringacc_ring_access_proxy(
+ ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem)
+{
+ return k3_nav_ringacc_ring_access_proxy(
+ ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_nav_ringacc_ring_access_io(
+ struct k3_nav_ring *ring, void *elem,
+ enum k3_ringacc_access_mode access_mode)
+{
+ void __iomem *ptr;
+
+ switch (access_mode) {
+ case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+ case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+ ptr = (void __iomem *)&ring->fifos->head_data;
+ break;
+ case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+ case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+ ptr = (void __iomem *)&ring->fifos->tail_data;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
+
+ switch (access_mode) {
+ case K3_RINGACC_ACCESS_MODE_POP_HEAD:
+ case K3_RINGACC_ACCESS_MODE_POP_TAIL:
+ pr_debug("memcpy_fromio(x): --> ptr(%p), mode:%d\n",
+ ptr, access_mode);
+ memcpy_fromio(elem, ptr, (4 << ring->elm_size));
+ ring->occ--;
+ break;
+ case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
+ case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
+ pr_debug("memcpy_toio(x): --> ptr(%p), mode:%d\n",
+ ptr, access_mode);
+ memcpy_toio(ptr, elem, (4 << ring->elm_size));
+ ring->free--;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ pr_debug("free%d index%d occ%d index%d\n",
+ ring->free, ring->windex, ring->occ, ring->rindex);
+ return 0;
+}
+
+static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
+ void *elem)
+{
+ return k3_nav_ringacc_ring_access_io(
+ ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
+}
+
+static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem)
+{
+ return k3_nav_ringacc_ring_access_io(
+ ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
+}
+
+static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem)
+{
+ return k3_nav_ringacc_ring_access_io(
+ ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring, void *elem)
+{
+ return k3_nav_ringacc_ring_access_io(
+ ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
+}
+
+static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
+{
+ void *elem_ptr;
+
+ elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->windex);
+
+ memcpy(elem_ptr, elem, (4 << ring->elm_size));
+
+ ring->windex = (ring->windex + 1) % ring->size;
+ ring->free--;
+ ringacc_writel(1, &ring->rt->db);
+
+ pr_debug("ring_push_mem: free%d index%d\n",
+ ring->free, ring->windex);
+
+ return 0;
+}
+
+static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
+{
+ void *elem_ptr;
+
+ elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->rindex);
+
+ memcpy(elem, elem_ptr, (4 << ring->elm_size));
+
+ ring->rindex = (ring->rindex + 1) % ring->size;
+ ring->occ--;
+ ringacc_writel(-1, &ring->rt->db);
+
+ pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
+ ring->occ, ring->rindex, elem_ptr);
+ return 0;
+}
+
+int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
+{
+ int ret = -EOPNOTSUPP;
+
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return -EINVAL;
+
+ pr_debug("ring_push%d: free%d index%d\n",
+ ring->ring_id, ring->free, ring->windex);
+
+ if (k3_nav_ringacc_ring_is_full(ring))
+ return -ENOMEM;
+
+ if (ring->ops && ring->ops->push_tail)
+ ret = ring->ops->push_tail(ring, elem);
+
+ return ret;
+}
+
+int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
+{
+ int ret = -EOPNOTSUPP;
+
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return -EINVAL;
+
+ pr_debug("ring_push_head: free%d index%d\n",
+ ring->free, ring->windex);
+
+ if (k3_nav_ringacc_ring_is_full(ring))
+ return -ENOMEM;
+
+ if (ring->ops && ring->ops->push_head)
+ ret = ring->ops->push_head(ring, elem);
+
+ return ret;
+}
+
+int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
+{
+ int ret = -EOPNOTSUPP;
+
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return -EINVAL;
+
+ if (!ring->occ)
+ ring->occ = k3_nav_ringacc_ring_get_occ(ring);
+
+ pr_debug("ring_pop%d: occ%d index%d\n",
+ ring->ring_id, ring->occ, ring->rindex);
+
+ if (!ring->occ)
+ return -ENODATA;
+
+ if (ring->ops && ring->ops->pop_head)
+ ret = ring->ops->pop_head(ring, elem);
+
+ return ret;
+}
+
+int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
+{
+ int ret = -EOPNOTSUPP;
+
+ if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
+ return -EINVAL;
+
+ if (!ring->occ)
+ ring->occ = k3_nav_ringacc_ring_get_occ(ring);
+
+ pr_debug("ring_pop_tail: occ%d index%d\n",
+ ring->occ, ring->rindex);
+
+ if (!ring->occ)
+ return -ENODATA;
+
+ if (ring->ops && ring->ops->pop_tail)
+ ret = ring->ops->pop_tail(ring, elem);
+
+ return ret;
+}
+
+static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
+{
+ struct udevice *dev = ringacc->dev;
+ struct udevice *tisci_dev = NULL;
+ int ret;
+
+ ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
+ if (!ringacc->num_rings) {
+ dev_err(dev, "ti,num-rings read failure %d\n", ret);
+ return -EINVAL;
+ }
+
+ ringacc->dma_ring_reset_quirk =
+ dev_read_bool(dev, "ti,dma-ring-reset-quirk");
+
+ ret = uclass_get_device_by_name(UCLASS_FIRMWARE, "dmsc", &tisci_dev);
+ if (ret) {
+ pr_debug("TISCI RA RM get failed (%d)\n", ret);
+ ringacc->tisci = NULL;
+ return -ENODEV;
+ }
+ ringacc->tisci = (struct ti_sci_handle *)
+ (ti_sci_get_handle_from_sysfw(tisci_dev));
+
+ ret = dev_read_u32_default(dev, "ti,sci", 0);
+ if (!ret) {
+ dev_err(dev, "TISCI RA RM disabled\n");
+ ringacc->tisci = NULL;
+ return ret;
+ }
+
+ ret = dev_read_u32(dev, "ti,sci-dev-id", &ringacc->tisci_dev_id);
+ if (ret) {
+ dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
+ ringacc->tisci = NULL;
+ return ret;
+ }
+
+ ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
+ ringacc->tisci, dev,
+ ringacc->tisci_dev_id,
+ "ti,sci-rm-range-gp-rings");
+ if (IS_ERR(ringacc->rm_gp_range))
+ ret = PTR_ERR(ringacc->rm_gp_range);
+
+ return 0;
+}
+
+static int k3_nav_ringacc_probe(struct udevice *dev)
+{
+ struct k3_nav_ringacc *ringacc;
+ void __iomem *base_fifo, *base_rt;
+ int ret, i;
+
+ ringacc = dev_get_priv(dev);
+ if (!ringacc)
+ return -ENOMEM;
+
+ ringacc->dev = dev;
+
+ ret = k3_nav_ringacc_probe_dt(ringacc);
+ if (ret)
+ return ret;
+
+ base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
+ pr_debug("rt %p\n", base_rt);
+ if (IS_ERR(base_rt))
+ return PTR_ERR(base_rt);
+
+ base_fifo = (uint32_t *)devfdt_get_addr_name(dev, "fifos");
+ pr_debug("fifos %p\n", base_fifo);
+ if (IS_ERR(base_fifo))
+ return PTR_ERR(base_fifo);
+
+ ringacc->proxy_gcfg = (struct k3_ringacc_proxy_gcfg_regs __iomem *)
+ devfdt_get_addr_name(dev, "proxy_gcfg");
+ if (IS_ERR(ringacc->proxy_gcfg))
+ return PTR_ERR(ringacc->proxy_gcfg);
+ ringacc->proxy_target_base =
+ (struct k3_ringacc_proxy_gcfg_regs __iomem *)
+ devfdt_get_addr_name(dev, "proxy_target");
+ if (IS_ERR(ringacc->proxy_target_base))
+ return PTR_ERR(ringacc->proxy_target_base);
+
+ ringacc->num_proxies = ringacc_readl(&ringacc->proxy_gcfg->config) &
+ K3_RINGACC_PROXY_CFG_THREADS_MASK;
+
+ ringacc->rings = devm_kzalloc(dev,
+ sizeof(*ringacc->rings) *
+ ringacc->num_rings,
+ GFP_KERNEL);
+ ringacc->rings_inuse = devm_kcalloc(dev,
+ BITS_TO_LONGS(ringacc->num_rings),
+ sizeof(unsigned long), GFP_KERNEL);
+ ringacc->proxy_inuse = devm_kcalloc(dev,
+ BITS_TO_LONGS(ringacc->num_proxies),
+ sizeof(unsigned long), GFP_KERNEL);
+
+ if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
+ return -ENOMEM;
+
+ for (i = 0; i < ringacc->num_rings; i++) {
+ ringacc->rings[i].rt = base_rt +
+ KNAV_RINGACC_RT_REGS_STEP * i;
+ ringacc->rings[i].fifos = base_fifo +
+ KNAV_RINGACC_FIFO_REGS_STEP * i;
+ ringacc->rings[i].parent = ringacc;
+ ringacc->rings[i].ring_id = i;
+ ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
+ }
+ dev_set_drvdata(dev, ringacc);
+
+ ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
+
+ list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
+
+ dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
+ ringacc->num_rings,
+ ringacc->rm_gp_range->desc[0].start,
+ ringacc->rm_gp_range->desc[0].num,
+ ringacc->tisci_dev_id);
+ dev_info(dev, "dma-ring-reset-quirk: %s\n",
+ ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
+ dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
+ ringacc_readl(&ringacc->proxy_gcfg->revision),
+ ringacc->num_proxies);
+ return 0;
+}
+
+static const struct udevice_id knav_ringacc_ids[] = {
+ { .compatible = "ti,am654-navss-ringacc" },
+ {},
+};
+
+U_BOOT_DRIVER(k3_navss_ringacc) = {
+ .name = "k3-navss-ringacc",
+ .id = UCLASS_MISC,
+ .of_match = knav_ringacc_ids,
+ .probe = k3_nav_ringacc_probe,
+ .priv_auto_alloc_size = sizeof(struct k3_nav_ringacc),
+};
projects / oweals / u-boot.git / blobdiff