drivers/soc/ti/k3-navss-ringacc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
   4  *
   5  * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
   6  */
   7
   8 #include <common.h>
   9 #include <cpu_func.h>
  10 #include <log.h>
  11 #include <asm/cache.h>
  12 #include <asm/io.h>
  13 #include <malloc.h>
  14 #include <asm/bitops.h>
  15 #include <dm.h>
  16 #include <dm/device_compat.h>
  17 #include <dm/devres.h>
  18 #include <dm/read.h>
  19 #include <dm/uclass.h>
  20 #include <linux/bitops.h>
  21 #include <linux/compat.h>
  22 #include <linux/dma-mapping.h>
  23 #include <linux/err.h>
  24 #include <linux/soc/ti/k3-navss-ringacc.h>
  25 #include <linux/soc/ti/ti_sci_protocol.h>
  26
  27 #define set_bit(bit, bitmap)    __set_bit(bit, bitmap)
  28 #define clear_bit(bit, bitmap)  __clear_bit(bit, bitmap)
  29 #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
  30         dma_free_coherent(cpu_addr)
  31 #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
  32 ({ \
  33         void    *ring_mem_virt; \
  34         ring_mem_virt = dma_alloc_coherent((size), \
  35                                            (unsigned long *)(dma_handle)); \
  36         if (ring_mem_virt) \
  37                 memset(ring_mem_virt, 0, (size)); \
  38         ring_mem_virt; \
  39 })
  40
  41 static LIST_HEAD(k3_nav_ringacc_list);
  42
  43 static  void ringacc_writel(u32 v, void __iomem *reg)
  44 {
  45         pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
  46         writel(v, reg);
  47 }
  48
  49 static  u32 ringacc_readl(void __iomem *reg)
  50 {
  51         u32 v;
  52
  53         v = readl(reg);
  54         pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
  55         return v;
  56 }
  57
  58 #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK           GENMASK(19, 0)
  59
  60 /**
  61  * struct k3_nav_ring_rt_regs -  The RA Control/Status Registers region
  62  */
  63 struct k3_nav_ring_rt_regs {
  64         u32     resv_16[4];
  65         u32     db;             /* RT Ring N Doorbell Register */
  66         u32     resv_4[1];
  67         u32     occ;            /* RT Ring N Occupancy Register */
  68         u32     indx;           /* RT Ring N Current Index Register */
  69         u32     hwocc;          /* RT Ring N Hardware Occupancy Register */
  70         u32     hwindx;         /* RT Ring N Current Index Register */
  71 };
  72
  73 #define KNAV_RINGACC_RT_REGS_STEP       0x1000
  74
  75 /**
  76  * struct k3_nav_ring_fifo_regs -  The Ring Accelerator Queues Registers region
  77  */
  78 struct k3_nav_ring_fifo_regs {
  79         u32     head_data[128];         /* Ring Head Entry Data Registers */
  80         u32     tail_data[128];         /* Ring Tail Entry Data Registers */
  81         u32     peek_head_data[128];    /* Ring Peek Head Entry Data Regs */
  82         u32     peek_tail_data[128];    /* Ring Peek Tail Entry Data Regs */
  83 };
  84
  85 /**
  86  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
  87  */
  88 struct k3_ringacc_proxy_gcfg_regs {
  89         u32     revision;       /* Revision Register */
  90         u32     config;         /* Config Register */
  91 };
  92
  93 #define K3_RINGACC_PROXY_CFG_THREADS_MASK               GENMASK(15, 0)
  94
  95 /**
  96  * struct k3_ringacc_proxy_target_regs -  RA Proxy Datapath MMIO Region
  97  */
  98 struct k3_ringacc_proxy_target_regs {
  99         u32     control;        /* Proxy Control Register */
 100         u32     status;         /* Proxy Status Register */
 101         u8      resv_512[504];
 102         u32     data[128];      /* Proxy Data Register */
 103 };
 104
 105 #define K3_RINGACC_PROXY_TARGET_STEP    0x1000
 106 #define K3_RINGACC_PROXY_NOT_USED       (-1)
 107
 108 enum k3_ringacc_proxy_access_mode {
 109         PROXY_ACCESS_MODE_HEAD = 0,
 110         PROXY_ACCESS_MODE_TAIL = 1,
 111         PROXY_ACCESS_MODE_PEEK_HEAD = 2,
 112         PROXY_ACCESS_MODE_PEEK_TAIL = 3,
 113 };
 114
 115 #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
 116 #define KNAV_RINGACC_FIFO_REGS_STEP     0x1000
 117 #define KNAV_RINGACC_MAX_DB_RING_CNT    (127U)
 118
 119 /**
 120  * struct k3_nav_ring_ops -  Ring operations
 121  */
 122 struct k3_nav_ring_ops {
 123         int (*push_tail)(struct k3_nav_ring *ring, void *elm);
 124         int (*push_head)(struct k3_nav_ring *ring, void *elm);
 125         int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
 126         int (*pop_head)(struct k3_nav_ring *ring, void *elm);
 127 };
 128
 129 /**
 130  * struct k3_nav_ring - RA Ring descriptor
 131  *
 132  * @rt - Ring control/status registers
 133  * @fifos - Ring queues registers
 134  * @proxy - Ring Proxy Datapath registers
 135  * @ring_mem_dma - Ring buffer dma address
 136  * @ring_mem_virt - Ring buffer virt address
 137  * @ops - Ring operations
 138  * @size - Ring size in elements
 139  * @elm_size - Size of the ring element
 140  * @mode - Ring mode
 141  * @flags - flags
 142  * @free - Number of free elements
 143  * @occ - Ring occupancy
 144  * @windex - Write index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 145  * @rindex - Read index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 146  * @ring_id - Ring Id
 147  * @parent - Pointer on struct @k3_nav_ringacc
 148  * @use_count - Use count for shared rings
 149  * @proxy_id - RA Ring Proxy Id (only if @K3_NAV_RINGACC_RING_USE_PROXY)
 150  */
 151 struct k3_nav_ring {
 152         struct k3_nav_ring_rt_regs __iomem *rt;
 153         struct k3_nav_ring_fifo_regs __iomem *fifos;
 154         struct k3_ringacc_proxy_target_regs  __iomem *proxy;
 155         dma_addr_t      ring_mem_dma;
 156         void            *ring_mem_virt;
 157         struct k3_nav_ring_ops *ops;
 158         u32             size;
 159         enum k3_nav_ring_size elm_size;
 160         enum k3_nav_ring_mode mode;
 161         u32             flags;
 162 #define KNAV_RING_FLAG_BUSY     BIT(1)
 163 #define K3_NAV_RING_FLAG_SHARED BIT(2)
 164         u32             free;
 165         u32             occ;
 166         u32             windex;
 167         u32             rindex;
 168         u32             ring_id;
 169         struct k3_nav_ringacc   *parent;
 170         u32             use_count;
 171         int             proxy_id;
 172 };
 173
 174 /**
 175  * struct k3_nav_ringacc - Rings accelerator descriptor
 176  *
 177  * @dev - pointer on RA device
 178  * @proxy_gcfg - RA proxy global config registers
 179  * @proxy_target_base - RA proxy datapath region
 180  * @num_rings - number of ring in RA
 181  * @rm_gp_range - general purpose rings range from tisci
 182  * @dma_ring_reset_quirk - DMA reset w/a enable
 183  * @num_proxies - number of RA proxies
 184  * @rings - array of rings descriptors (struct @k3_nav_ring)
 185  * @list - list of RAs in the system
 186  * @tisci - pointer ti-sci handle
 187  * @tisci_ring_ops - ti-sci rings ops
 188  * @tisci_dev_id - ti-sci device id
 189  */
 190 struct k3_nav_ringacc {
 191         struct udevice *dev;
 192         struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
 193         void __iomem *proxy_target_base;
 194         u32 num_rings; /* number of rings in Ringacc module */
 195         unsigned long *rings_inuse;
 196         struct ti_sci_resource *rm_gp_range;
 197         bool dma_ring_reset_quirk;
 198         u32 num_proxies;
 199         unsigned long *proxy_inuse;
 200
 201         struct k3_nav_ring *rings;
 202         struct list_head list;
 203
 204         const struct ti_sci_handle *tisci;
 205         const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
 206         u32  tisci_dev_id;
 207 };
 208
 209 static long k3_nav_ringacc_ring_get_fifo_pos(struct k3_nav_ring *ring)
 210 {
 211         return KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES -
 212                (4 << ring->elm_size);
 213 }
 214
 215 static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
 216 {
 217         return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
 218 }
 219
 220 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
 221 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
 222
 223 static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
 224                 .push_tail = k3_nav_ringacc_ring_push_mem,
 225                 .pop_head = k3_nav_ringacc_ring_pop_mem,
 226 };
 227
 228 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem);
 229 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem);
 230 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 231                                             void *elem);
 232 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring,
 233                                            void *elem);
 234
 235 static struct k3_nav_ring_ops k3_nav_mode_msg_ops = {
 236                 .push_tail = k3_nav_ringacc_ring_push_io,
 237                 .push_head = k3_nav_ringacc_ring_push_head_io,
 238                 .pop_tail = k3_nav_ringacc_ring_pop_tail_io,
 239                 .pop_head = k3_nav_ringacc_ring_pop_io,
 240 };
 241
 242 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring,
 243                                            void *elem);
 244 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring,
 245                                            void *elem);
 246 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem);
 247 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem);
 248
 249 static struct k3_nav_ring_ops k3_nav_mode_proxy_ops = {
 250                 .push_tail = k3_ringacc_ring_push_tail_proxy,
 251                 .push_head = k3_ringacc_ring_push_head_proxy,
 252                 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
 253                 .pop_head = k3_ringacc_ring_pop_head_proxy,
 254 };
 255
 256 struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
 257 {
 258         return ringacc->dev;
 259 }
 260
 261 struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
 262                                                 int id, u32 flags)
 263 {
 264         int proxy_id = K3_RINGACC_PROXY_NOT_USED;
 265
 266         if (id == K3_NAV_RINGACC_RING_ID_ANY) {
 267                 /* Request for any general purpose ring */
 268                 struct ti_sci_resource_desc *gp_rings =
 269                                         &ringacc->rm_gp_range->desc[0];
 270                 unsigned long size;
 271
 272                 size = gp_rings->start + gp_rings->num;
 273                 id = find_next_zero_bit(ringacc->rings_inuse,
 274                                         size, gp_rings->start);
 275                 if (id == size)
 276                         goto error;
 277         } else if (id < 0) {
 278                 goto error;
 279         }
 280
 281         if (test_bit(id, ringacc->rings_inuse) &&
 282             !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
 283                 goto error;
 284         else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
 285                 goto out;
 286
 287         if (flags & K3_NAV_RINGACC_RING_USE_PROXY) {
 288                 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
 289                                               ringacc->num_proxies, 0);
 290                 if (proxy_id == ringacc->num_proxies)
 291                         goto error;
 292         }
 293
 294         if (!try_module_get(ringacc->dev->driver->owner))
 295                 goto error;
 296
 297         if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 298                 set_bit(proxy_id, ringacc->proxy_inuse);
 299                 ringacc->rings[id].proxy_id = proxy_id;
 300                 pr_debug("Giving ring#%d proxy#%d\n",
 301                          id, proxy_id);
 302         } else {
 303                 pr_debug("Giving ring#%d\n", id);
 304         }
 305
 306         set_bit(id, ringacc->rings_inuse);
 307 out:
 308         ringacc->rings[id].use_count++;
 309         return &ringacc->rings[id];
 310
 311 error:
 312         return NULL;
 313 }
 314
 315 static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
 316 {
 317         struct k3_nav_ringacc *ringacc = ring->parent;
 318         int ret;
 319
 320         ret = ringacc->tisci_ring_ops->config(
 321                         ringacc->tisci,
 322                         TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
 323                         ringacc->tisci_dev_id,
 324                         ring->ring_id,
 325                         0,
 326                         0,
 327                         ring->size,
 328                         0,
 329                         0,
 330                         0);
 331         if (ret)
 332                 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
 333                         ret, ring->ring_id);
 334 }
 335
 336 void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
 337 {
 338         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 339                 return;
 340
 341         ring->occ = 0;
 342         ring->free = 0;
 343         ring->rindex = 0;
 344         ring->windex = 0;
 345
 346         k3_ringacc_ring_reset_sci(ring);
 347 }
 348
 349 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
 350                                                enum k3_nav_ring_mode mode)
 351 {
 352         struct k3_nav_ringacc *ringacc = ring->parent;
 353         int ret;
 354
 355         ret = ringacc->tisci_ring_ops->config(
 356                         ringacc->tisci,
 357                         TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
 358                         ringacc->tisci_dev_id,
 359                         ring->ring_id,
 360                         0,
 361                         0,
 362                         0,
 363                         mode,
 364                         0,
 365                         0);
 366         if (ret)
 367                 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
 368                         ret, ring->ring_id);
 369 }
 370
 371 void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
 372 {
 373         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 374                 return;
 375
 376         if (!ring->parent->dma_ring_reset_quirk) {
 377                 k3_nav_ringacc_ring_reset(ring);
 378                 return;
 379         }
 380
 381         if (!occ)
 382                 occ = ringacc_readl(&ring->rt->occ);
 383
 384         if (occ) {
 385                 u32 db_ring_cnt, db_ring_cnt_cur;
 386
 387                 pr_debug("%s %u occ: %u\n", __func__,
 388                          ring->ring_id, occ);
 389                 /* 2. Reset the ring */
 390                 k3_ringacc_ring_reset_sci(ring);
 391
 392                 /*
 393                  * 3. Setup the ring in ring/doorbell mode
 394                  * (if not already in this mode)
 395                  */
 396                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 397                         k3_ringacc_ring_reconfig_qmode_sci(
 398                                         ring, K3_NAV_RINGACC_RING_MODE_RING);
 399                 /*
 400                  * 4. Ring the doorbell 2**22 – ringOcc times.
 401                  * This will wrap the internal UDMAP ring state occupancy
 402                  * counter (which is 21-bits wide) to 0.
 403                  */
 404                 db_ring_cnt = (1U << 22) - occ;
 405
 406                 while (db_ring_cnt != 0) {
 407                         /*
 408                          * Ring the doorbell with the maximum count each
 409                          * iteration if possible to minimize the total
 410                          * of writes
 411                          */
 412                         if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
 413                                 db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
 414                         else
 415                                 db_ring_cnt_cur = db_ring_cnt;
 416
 417                         writel(db_ring_cnt_cur, &ring->rt->db);
 418                         db_ring_cnt -= db_ring_cnt_cur;
 419                 }
 420
 421                 /* 5. Restore the original ring mode (if not ring mode) */
 422                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 423                         k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
 424         }
 425
 426         /* 2. Reset the ring */
 427         k3_nav_ringacc_ring_reset(ring);
 428 }
 429
 430 static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
 431 {
 432         struct k3_nav_ringacc *ringacc = ring->parent;
 433         int ret;
 434
 435         ret = ringacc->tisci_ring_ops->config(
 436                         ringacc->tisci,
 437                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 438                         ringacc->tisci_dev_id,
 439                         ring->ring_id,
 440                         0,
 441                         0,
 442                         0,
 443                         0,
 444                         0,
 445                         0);
 446         if (ret)
 447                 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
 448                         ret, ring->ring_id);
 449 }
 450
 451 int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
 452 {
 453         struct k3_nav_ringacc *ringacc;
 454
 455         if (!ring)
 456                 return -EINVAL;
 457
 458         ringacc = ring->parent;
 459
 460         pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
 461
 462         if (!test_bit(ring->ring_id, ringacc->rings_inuse))
 463                 return -EINVAL;
 464
 465         if (--ring->use_count)
 466                 goto out;
 467
 468         if (!(ring->flags & KNAV_RING_FLAG_BUSY))
 469                 goto no_init;
 470
 471         k3_ringacc_ring_free_sci(ring);
 472
 473         dma_free_coherent(ringacc->dev,
 474                           ring->size * (4 << ring->elm_size),
 475                           ring->ring_mem_virt, ring->ring_mem_dma);
 476         ring->flags &= ~KNAV_RING_FLAG_BUSY;
 477         ring->ops = NULL;
 478         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 479                 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
 480                 ring->proxy = NULL;
 481                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
 482         }
 483
 484 no_init:
 485         clear_bit(ring->ring_id, ringacc->rings_inuse);
 486
 487         module_put(ringacc->dev->driver->owner);
 488
 489 out:
 490         return 0;
 491 }
 492
 493 u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
 494 {
 495         if (!ring)
 496                 return -EINVAL;
 497
 498         return ring->ring_id;
 499 }
 500
 501 static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
 502 {
 503         struct k3_nav_ringacc *ringacc = ring->parent;
 504         u32 ring_idx;
 505         int ret;
 506
 507         if (!ringacc->tisci)
 508                 return -EINVAL;
 509
 510         ring_idx = ring->ring_id;
 511         ret = ringacc->tisci_ring_ops->config(
 512                         ringacc->tisci,
 513                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 514                         ringacc->tisci_dev_id,
 515                         ring_idx,
 516                         lower_32_bits(ring->ring_mem_dma),
 517                         upper_32_bits(ring->ring_mem_dma),
 518                         ring->size,
 519                         ring->mode,
 520                         ring->elm_size,
 521                         0);
 522         if (ret)
 523                 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
 524                         ret, ring_idx);
 525
 526         return ret;
 527 }
 528
 529 int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
 530                             struct k3_nav_ring_cfg *cfg)
 531 {
 532         struct k3_nav_ringacc *ringacc = ring->parent;
 533         int ret = 0;
 534
 535         if (!ring || !cfg)
 536                 return -EINVAL;
 537         if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
 538             cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
 539             cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
 540             !test_bit(ring->ring_id, ringacc->rings_inuse))
 541                 return -EINVAL;
 542
 543         if (ring->use_count != 1)
 544                 return 0;
 545
 546         ring->size = cfg->size;
 547         ring->elm_size = cfg->elm_size;
 548         ring->mode = cfg->mode;
 549         ring->occ = 0;
 550         ring->free = 0;
 551         ring->rindex = 0;
 552         ring->windex = 0;
 553
 554         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
 555                 ring->proxy = ringacc->proxy_target_base +
 556                               ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
 557
 558         switch (ring->mode) {
 559         case K3_NAV_RINGACC_RING_MODE_RING:
 560                 ring->ops = &k3_nav_mode_ring_ops;
 561                 break;
 562         case K3_NAV_RINGACC_RING_MODE_QM:
 563                 /*
 564                  * In Queue mode elm_size can be 8 only and each operation
 565                  * uses 2 element slots
 566                  */
 567                 if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
 568                     cfg->size % 2)
 569                         goto err_free_proxy;
 570         case K3_NAV_RINGACC_RING_MODE_MESSAGE:
 571                 if (ring->proxy)
 572                         ring->ops = &k3_nav_mode_proxy_ops;
 573                 else
 574                         ring->ops = &k3_nav_mode_msg_ops;
 575                 break;
 576         default:
 577                 ring->ops = NULL;
 578                 ret = -EINVAL;
 579                 goto err_free_proxy;
 580         };
 581
 582         ring->ring_mem_virt =
 583                         dma_zalloc_coherent(ringacc->dev,
 584                                             ring->size * (4 << ring->elm_size),
 585                                             &ring->ring_mem_dma, GFP_KERNEL);
 586         if (!ring->ring_mem_virt) {
 587                 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
 588                 ret = -ENOMEM;
 589                 goto err_free_ops;
 590         }
 591
 592         ret = k3_nav_ringacc_ring_cfg_sci(ring);
 593
 594         if (ret)
 595                 goto err_free_mem;
 596
 597         ring->flags |= KNAV_RING_FLAG_BUSY;
 598         ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
 599                         K3_NAV_RING_FLAG_SHARED : 0;
 600
 601         return 0;
 602
 603 err_free_mem:
 604         dma_free_coherent(ringacc->dev,
 605                           ring->size * (4 << ring->elm_size),
 606                           ring->ring_mem_virt,
 607                           ring->ring_mem_dma);
 608 err_free_ops:
 609         ring->ops = NULL;
 610 err_free_proxy:
 611         ring->proxy = NULL;
 612         return ret;
 613 }
 614
 615 u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
 616 {
 617         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 618                 return -EINVAL;
 619
 620         return ring->size;
 621 }
 622
 623 u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
 624 {
 625         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 626                 return -EINVAL;
 627
 628         if (!ring->free)
 629                 ring->free = ring->size - ringacc_readl(&ring->rt->occ);
 630
 631         return ring->free;
 632 }
 633
 634 u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
 635 {
 636         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 637                 return -EINVAL;
 638
 639         return ringacc_readl(&ring->rt->occ);
 640 }
 641
 642 u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
 643 {
 644         return !k3_nav_ringacc_ring_get_free(ring);
 645 }
 646
 647 enum k3_ringacc_access_mode {
 648         K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
 649         K3_RINGACC_ACCESS_MODE_POP_HEAD,
 650         K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
 651         K3_RINGACC_ACCESS_MODE_POP_TAIL,
 652         K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
 653         K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
 654 };
 655
 656 static int k3_ringacc_ring_cfg_proxy(struct k3_nav_ring *ring,
 657                                      enum k3_ringacc_proxy_access_mode mode)
 658 {
 659         u32 val;
 660
 661         val = ring->ring_id;
 662         val |= mode << 16;
 663         val |= ring->elm_size << 24;
 664         ringacc_writel(val, &ring->proxy->control);
 665         return 0;
 666 }
 667
 668 static int k3_nav_ringacc_ring_access_proxy(
 669                         struct k3_nav_ring *ring, void *elem,
 670                         enum k3_ringacc_access_mode access_mode)
 671 {
 672         void __iomem *ptr;
 673
 674         ptr = (void __iomem *)&ring->proxy->data;
 675
 676         switch (access_mode) {
 677         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 678         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 679                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
 680                 break;
 681         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 682         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 683                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
 684                 break;
 685         default:
 686                 return -EINVAL;
 687         }
 688
 689         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 690
 691         switch (access_mode) {
 692         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 693         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 694                 pr_debug("proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 695                          ptr, access_mode);
 696                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 697                 ring->occ--;
 698                 break;
 699         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 700         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 701                 pr_debug("proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
 702                          ptr, access_mode);
 703                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 704                 ring->free--;
 705                 break;
 706         default:
 707                 return -EINVAL;
 708         }
 709
 710         pr_debug("proxy: free%d occ%d\n",
 711                  ring->free, ring->occ);
 712         return 0;
 713 }
 714
 715 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring, void *elem)
 716 {
 717         return k3_nav_ringacc_ring_access_proxy(
 718                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 719 }
 720
 721 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring, void *elem)
 722 {
 723         return k3_nav_ringacc_ring_access_proxy(
 724                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 725 }
 726
 727 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem)
 728 {
 729         return k3_nav_ringacc_ring_access_proxy(
 730                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 731 }
 732
 733 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem)
 734 {
 735         return k3_nav_ringacc_ring_access_proxy(
 736                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 737 }
 738
 739 static int k3_nav_ringacc_ring_access_io(
 740                 struct k3_nav_ring *ring, void *elem,
 741                 enum k3_ringacc_access_mode access_mode)
 742 {
 743         void __iomem *ptr;
 744
 745         switch (access_mode) {
 746         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 747         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 748                 ptr = (void __iomem *)&ring->fifos->head_data;
 749                 break;
 750         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 751         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 752                 ptr = (void __iomem *)&ring->fifos->tail_data;
 753                 break;
 754         default:
 755                 return -EINVAL;
 756         }
 757
 758         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 759
 760         switch (access_mode) {
 761         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 762         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 763                 pr_debug("memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 764                          ptr, access_mode);
 765                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 766                 ring->occ--;
 767                 break;
 768         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 769         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 770                 pr_debug("memcpy_toio(x): --> ptr(%p), mode:%d\n",
 771                          ptr, access_mode);
 772                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 773                 ring->free--;
 774                 break;
 775         default:
 776                 return -EINVAL;
 777         }
 778
 779         pr_debug("free%d index%d occ%d index%d\n",
 780                  ring->free, ring->windex, ring->occ, ring->rindex);
 781         return 0;
 782 }
 783
 784 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 785                                             void *elem)
 786 {
 787         return k3_nav_ringacc_ring_access_io(
 788                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 789 }
 790
 791 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem)
 792 {
 793         return k3_nav_ringacc_ring_access_io(
 794                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 795 }
 796
 797 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem)
 798 {
 799         return k3_nav_ringacc_ring_access_io(
 800                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 801 }
 802
 803 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring, void *elem)
 804 {
 805         return k3_nav_ringacc_ring_access_io(
 806                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 807 }
 808
 809 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
 810 {
 811         void *elem_ptr;
 812
 813         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->windex);
 814
 815         memcpy(elem_ptr, elem, (4 << ring->elm_size));
 816
 817         flush_dcache_range((unsigned long)ring->ring_mem_virt,
 818                            ALIGN((unsigned long)ring->ring_mem_virt +
 819                                  ring->size * (4 << ring->elm_size),
 820                                  ARCH_DMA_MINALIGN));
 821
 822         ring->windex = (ring->windex + 1) % ring->size;
 823         ring->free--;
 824         ringacc_writel(1, &ring->rt->db);
 825
 826         pr_debug("ring_push_mem: free%d index%d\n",
 827                  ring->free, ring->windex);
 828
 829         return 0;
 830 }
 831
 832 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
 833 {
 834         void *elem_ptr;
 835
 836         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->rindex);
 837
 838         invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
 839                                 ALIGN((unsigned long)ring->ring_mem_virt +
 840                                       ring->size * (4 << ring->elm_size),
 841                                       ARCH_DMA_MINALIGN));
 842
 843         memcpy(elem, elem_ptr, (4 << ring->elm_size));
 844
 845         ring->rindex = (ring->rindex + 1) % ring->size;
 846         ring->occ--;
 847         ringacc_writel(-1, &ring->rt->db);
 848
 849         pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
 850                  ring->occ, ring->rindex, elem_ptr);
 851         return 0;
 852 }
 853
 854 int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
 855 {
 856         int ret = -EOPNOTSUPP;
 857
 858         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 859                 return -EINVAL;
 860
 861         pr_debug("ring_push%d: free%d index%d\n",
 862                  ring->ring_id, ring->free, ring->windex);
 863
 864         if (k3_nav_ringacc_ring_is_full(ring))
 865                 return -ENOMEM;
 866
 867         if (ring->ops && ring->ops->push_tail)
 868                 ret = ring->ops->push_tail(ring, elem);
 869
 870         return ret;
 871 }
 872
 873 int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
 874 {
 875         int ret = -EOPNOTSUPP;
 876
 877         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 878                 return -EINVAL;
 879
 880         pr_debug("ring_push_head: free%d index%d\n",
 881                  ring->free, ring->windex);
 882
 883         if (k3_nav_ringacc_ring_is_full(ring))
 884                 return -ENOMEM;
 885
 886         if (ring->ops && ring->ops->push_head)
 887                 ret = ring->ops->push_head(ring, elem);
 888
 889         return ret;
 890 }
 891
 892 int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
 893 {
 894         int ret = -EOPNOTSUPP;
 895
 896         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 897                 return -EINVAL;
 898
 899         if (!ring->occ)
 900                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 901
 902         pr_debug("ring_pop%d: occ%d index%d\n",
 903                  ring->ring_id, ring->occ, ring->rindex);
 904
 905         if (!ring->occ)
 906                 return -ENODATA;
 907
 908         if (ring->ops && ring->ops->pop_head)
 909                 ret = ring->ops->pop_head(ring, elem);
 910
 911         return ret;
 912 }
 913
 914 int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
 915 {
 916         int ret = -EOPNOTSUPP;
 917
 918         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 919                 return -EINVAL;
 920
 921         if (!ring->occ)
 922                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 923
 924         pr_debug("ring_pop_tail: occ%d index%d\n",
 925                  ring->occ, ring->rindex);
 926
 927         if (!ring->occ)
 928                 return -ENODATA;
 929
 930         if (ring->ops && ring->ops->pop_tail)
 931                 ret = ring->ops->pop_tail(ring, elem);
 932
 933         return ret;
 934 }
 935
 936 static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
 937 {
 938         struct udevice *dev = ringacc->dev;
 939         struct udevice *tisci_dev = NULL;
 940         int ret;
 941
 942         ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
 943         if (!ringacc->num_rings) {
 944                 dev_err(dev, "ti,num-rings read failure %d\n", ret);
 945                 return -EINVAL;
 946         }
 947
 948         ringacc->dma_ring_reset_quirk =
 949                         dev_read_bool(dev, "ti,dma-ring-reset-quirk");
 950
 951         ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, dev,
 952                                            "ti,sci", &tisci_dev);
 953         if (ret) {
 954                 pr_debug("TISCI RA RM get failed (%d)\n", ret);
 955                 ringacc->tisci = NULL;
 956                 return -ENODEV;
 957         }
 958         ringacc->tisci = (struct ti_sci_handle *)
 959                          (ti_sci_get_handle_from_sysfw(tisci_dev));
 960
 961         ret = dev_read_u32_default(dev, "ti,sci", 0);
 962         if (!ret) {
 963                 dev_err(dev, "TISCI RA RM disabled\n");
 964                 ringacc->tisci = NULL;
 965                 return ret;
 966         }
 967
 968         ret = dev_read_u32(dev, "ti,sci-dev-id", &ringacc->tisci_dev_id);
 969         if (ret) {
 970                 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
 971                 ringacc->tisci = NULL;
 972                 return ret;
 973         }
 974
 975         ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
 976                                         ringacc->tisci, dev,
 977                                         ringacc->tisci_dev_id,
 978                                         "ti,sci-rm-range-gp-rings");
 979         if (IS_ERR(ringacc->rm_gp_range))
 980                 ret = PTR_ERR(ringacc->rm_gp_range);
 981
 982         return 0;
 983 }
 984
 985 static int k3_nav_ringacc_probe(struct udevice *dev)
 986 {
 987         struct k3_nav_ringacc *ringacc;
 988         void __iomem *base_fifo, *base_rt;
 989         int ret, i;
 990
 991         ringacc = dev_get_priv(dev);
 992         if (!ringacc)
 993                 return -ENOMEM;
 994
 995         ringacc->dev = dev;
 996
 997         ret = k3_nav_ringacc_probe_dt(ringacc);
 998         if (ret)
 999                 return ret;
1000
1001         base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
1002         pr_debug("rt %p\n", base_rt);
1003         if (IS_ERR(base_rt))
1004                 return PTR_ERR(base_rt);
1005
1006         base_fifo = (uint32_t *)devfdt_get_addr_name(dev, "fifos");
1007         pr_debug("fifos %p\n", base_fifo);
1008         if (IS_ERR(base_fifo))
1009                 return PTR_ERR(base_fifo);
1010
1011         ringacc->proxy_gcfg = (struct k3_ringacc_proxy_gcfg_regs __iomem *)
1012                 devfdt_get_addr_name(dev, "proxy_gcfg");
1013         if (IS_ERR(ringacc->proxy_gcfg))
1014                 return PTR_ERR(ringacc->proxy_gcfg);
1015         ringacc->proxy_target_base =
1016                 (struct k3_ringacc_proxy_gcfg_regs __iomem *)
1017                 devfdt_get_addr_name(dev, "proxy_target");
1018         if (IS_ERR(ringacc->proxy_target_base))
1019                 return PTR_ERR(ringacc->proxy_target_base);
1020
1021         ringacc->num_proxies = ringacc_readl(&ringacc->proxy_gcfg->config) &
1022                                          K3_RINGACC_PROXY_CFG_THREADS_MASK;
1023
1024         ringacc->rings = devm_kzalloc(dev,
1025                                       sizeof(*ringacc->rings) *
1026                                       ringacc->num_rings,
1027                                       GFP_KERNEL);
1028         ringacc->rings_inuse = devm_kcalloc(dev,
1029                                             BITS_TO_LONGS(ringacc->num_rings),
1030                                             sizeof(unsigned long), GFP_KERNEL);
1031         ringacc->proxy_inuse = devm_kcalloc(dev,
1032                                             BITS_TO_LONGS(ringacc->num_proxies),
1033                                             sizeof(unsigned long), GFP_KERNEL);
1034
1035         if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1036                 return -ENOMEM;
1037
1038         for (i = 0; i < ringacc->num_rings; i++) {
1039                 ringacc->rings[i].rt = base_rt +
1040                                        KNAV_RINGACC_RT_REGS_STEP * i;
1041                 ringacc->rings[i].fifos = base_fifo +
1042                                           KNAV_RINGACC_FIFO_REGS_STEP * i;
1043                 ringacc->rings[i].parent = ringacc;
1044                 ringacc->rings[i].ring_id = i;
1045                 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1046         }
1047         dev_set_drvdata(dev, ringacc);
1048
1049         ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1050
1051         list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
1052
1053         dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1054                  ringacc->num_rings,
1055                  ringacc->rm_gp_range->desc[0].start,
1056                  ringacc->rm_gp_range->desc[0].num,
1057                  ringacc->tisci_dev_id);
1058         dev_info(dev, "dma-ring-reset-quirk: %s\n",
1059                  ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1060         dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1061                  ringacc_readl(&ringacc->proxy_gcfg->revision),
1062                  ringacc->num_proxies);
1063         return 0;
1064 }
1065
1066 static const struct udevice_id knav_ringacc_ids[] = {
1067         { .compatible = "ti,am654-navss-ringacc" },
1068         {},
1069 };
1070
1071 U_BOOT_DRIVER(k3_navss_ringacc) = {
1072         .name   = "k3-navss-ringacc",
1073         .id     = UCLASS_MISC,
1074         .of_match = knav_ringacc_ids,
1075         .probe = k3_nav_ringacc_probe,
1076         .priv_auto_alloc_size = sizeof(struct k3_nav_ringacc),
1077 };