drivers/soc/ti/k3-navss-ringacc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
   4  *
   5  * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
   6  */
   7
   8 #include <common.h>
   9 #include <cpu_func.h>
  10 #include <asm/io.h>
  11 #include <malloc.h>
  12 #include <asm/dma-mapping.h>
  13 #include <asm/bitops.h>
  14 #include <dm.h>
  15 #include <dm/devres.h>
  16 #include <dm/read.h>
  17 #include <dm/uclass.h>
  18 #include <linux/compat.h>
  19 #include <linux/err.h>
  20 #include <linux/soc/ti/k3-navss-ringacc.h>
  21 #include <linux/soc/ti/ti_sci_protocol.h>
  22
  23 #define set_bit(bit, bitmap)    __set_bit(bit, bitmap)
  24 #define clear_bit(bit, bitmap)  __clear_bit(bit, bitmap)
  25 #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
  26         dma_free_coherent(cpu_addr)
  27 #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
  28 ({ \
  29         void    *ring_mem_virt; \
  30         ring_mem_virt = dma_alloc_coherent((size), \
  31                                            (unsigned long *)(dma_handle)); \
  32         if (ring_mem_virt) \
  33                 memset(ring_mem_virt, 0, (size)); \
  34         ring_mem_virt; \
  35 })
  36
  37 static LIST_HEAD(k3_nav_ringacc_list);
  38
  39 static  void ringacc_writel(u32 v, void __iomem *reg)
  40 {
  41         pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
  42         writel(v, reg);
  43 }
  44
  45 static  u32 ringacc_readl(void __iomem *reg)
  46 {
  47         u32 v;
  48
  49         v = readl(reg);
  50         pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
  51         return v;
  52 }
  53
  54 #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK           GENMASK(19, 0)
  55
  56 /**
  57  * struct k3_nav_ring_rt_regs -  The RA Control/Status Registers region
  58  */
  59 struct k3_nav_ring_rt_regs {
  60         u32     resv_16[4];
  61         u32     db;             /* RT Ring N Doorbell Register */
  62         u32     resv_4[1];
  63         u32     occ;            /* RT Ring N Occupancy Register */
  64         u32     indx;           /* RT Ring N Current Index Register */
  65         u32     hwocc;          /* RT Ring N Hardware Occupancy Register */
  66         u32     hwindx;         /* RT Ring N Current Index Register */
  67 };
  68
  69 #define KNAV_RINGACC_RT_REGS_STEP       0x1000
  70
  71 /**
  72  * struct k3_nav_ring_fifo_regs -  The Ring Accelerator Queues Registers region
  73  */
  74 struct k3_nav_ring_fifo_regs {
  75         u32     head_data[128];         /* Ring Head Entry Data Registers */
  76         u32     tail_data[128];         /* Ring Tail Entry Data Registers */
  77         u32     peek_head_data[128];    /* Ring Peek Head Entry Data Regs */
  78         u32     peek_tail_data[128];    /* Ring Peek Tail Entry Data Regs */
  79 };
  80
  81 /**
  82  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
  83  */
  84 struct k3_ringacc_proxy_gcfg_regs {
  85         u32     revision;       /* Revision Register */
  86         u32     config;         /* Config Register */
  87 };
  88
  89 #define K3_RINGACC_PROXY_CFG_THREADS_MASK               GENMASK(15, 0)
  90
  91 /**
  92  * struct k3_ringacc_proxy_target_regs -  RA Proxy Datapath MMIO Region
  93  */
  94 struct k3_ringacc_proxy_target_regs {
  95         u32     control;        /* Proxy Control Register */
  96         u32     status;         /* Proxy Status Register */
  97         u8      resv_512[504];
  98         u32     data[128];      /* Proxy Data Register */
  99 };
 100
 101 #define K3_RINGACC_PROXY_TARGET_STEP    0x1000
 102 #define K3_RINGACC_PROXY_NOT_USED       (-1)
 103
 104 enum k3_ringacc_proxy_access_mode {
 105         PROXY_ACCESS_MODE_HEAD = 0,
 106         PROXY_ACCESS_MODE_TAIL = 1,
 107         PROXY_ACCESS_MODE_PEEK_HEAD = 2,
 108         PROXY_ACCESS_MODE_PEEK_TAIL = 3,
 109 };
 110
 111 #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
 112 #define KNAV_RINGACC_FIFO_REGS_STEP     0x1000
 113 #define KNAV_RINGACC_MAX_DB_RING_CNT    (127U)
 114
 115 /**
 116  * struct k3_nav_ring_ops -  Ring operations
 117  */
 118 struct k3_nav_ring_ops {
 119         int (*push_tail)(struct k3_nav_ring *ring, void *elm);
 120         int (*push_head)(struct k3_nav_ring *ring, void *elm);
 121         int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
 122         int (*pop_head)(struct k3_nav_ring *ring, void *elm);
 123 };
 124
 125 /**
 126  * struct k3_nav_ring - RA Ring descriptor
 127  *
 128  * @rt - Ring control/status registers
 129  * @fifos - Ring queues registers
 130  * @proxy - Ring Proxy Datapath registers
 131  * @ring_mem_dma - Ring buffer dma address
 132  * @ring_mem_virt - Ring buffer virt address
 133  * @ops - Ring operations
 134  * @size - Ring size in elements
 135  * @elm_size - Size of the ring element
 136  * @mode - Ring mode
 137  * @flags - flags
 138  * @free - Number of free elements
 139  * @occ - Ring occupancy
 140  * @windex - Write index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 141  * @rindex - Read index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 142  * @ring_id - Ring Id
 143  * @parent - Pointer on struct @k3_nav_ringacc
 144  * @use_count - Use count for shared rings
 145  * @proxy_id - RA Ring Proxy Id (only if @K3_NAV_RINGACC_RING_USE_PROXY)
 146  */
 147 struct k3_nav_ring {
 148         struct k3_nav_ring_rt_regs __iomem *rt;
 149         struct k3_nav_ring_fifo_regs __iomem *fifos;
 150         struct k3_ringacc_proxy_target_regs  __iomem *proxy;
 151         dma_addr_t      ring_mem_dma;
 152         void            *ring_mem_virt;
 153         struct k3_nav_ring_ops *ops;
 154         u32             size;
 155         enum k3_nav_ring_size elm_size;
 156         enum k3_nav_ring_mode mode;
 157         u32             flags;
 158 #define KNAV_RING_FLAG_BUSY     BIT(1)
 159 #define K3_NAV_RING_FLAG_SHARED BIT(2)
 160         u32             free;
 161         u32             occ;
 162         u32             windex;
 163         u32             rindex;
 164         u32             ring_id;
 165         struct k3_nav_ringacc   *parent;
 166         u32             use_count;
 167         int             proxy_id;
 168 };
 169
 170 /**
 171  * struct k3_nav_ringacc - Rings accelerator descriptor
 172  *
 173  * @dev - pointer on RA device
 174  * @proxy_gcfg - RA proxy global config registers
 175  * @proxy_target_base - RA proxy datapath region
 176  * @num_rings - number of ring in RA
 177  * @rm_gp_range - general purpose rings range from tisci
 178  * @dma_ring_reset_quirk - DMA reset w/a enable
 179  * @num_proxies - number of RA proxies
 180  * @rings - array of rings descriptors (struct @k3_nav_ring)
 181  * @list - list of RAs in the system
 182  * @tisci - pointer ti-sci handle
 183  * @tisci_ring_ops - ti-sci rings ops
 184  * @tisci_dev_id - ti-sci device id
 185  */
 186 struct k3_nav_ringacc {
 187         struct udevice *dev;
 188         struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
 189         void __iomem *proxy_target_base;
 190         u32 num_rings; /* number of rings in Ringacc module */
 191         unsigned long *rings_inuse;
 192         struct ti_sci_resource *rm_gp_range;
 193         bool dma_ring_reset_quirk;
 194         u32 num_proxies;
 195         unsigned long *proxy_inuse;
 196
 197         struct k3_nav_ring *rings;
 198         struct list_head list;
 199
 200         const struct ti_sci_handle *tisci;
 201         const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
 202         u32  tisci_dev_id;
 203 };
 204
 205 static long k3_nav_ringacc_ring_get_fifo_pos(struct k3_nav_ring *ring)
 206 {
 207         return KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES -
 208                (4 << ring->elm_size);
 209 }
 210
 211 static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
 212 {
 213         return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
 214 }
 215
 216 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
 217 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
 218
 219 static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
 220                 .push_tail = k3_nav_ringacc_ring_push_mem,
 221                 .pop_head = k3_nav_ringacc_ring_pop_mem,
 222 };
 223
 224 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem);
 225 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem);
 226 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 227                                             void *elem);
 228 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring,
 229                                            void *elem);
 230
 231 static struct k3_nav_ring_ops k3_nav_mode_msg_ops = {
 232                 .push_tail = k3_nav_ringacc_ring_push_io,
 233                 .push_head = k3_nav_ringacc_ring_push_head_io,
 234                 .pop_tail = k3_nav_ringacc_ring_pop_tail_io,
 235                 .pop_head = k3_nav_ringacc_ring_pop_io,
 236 };
 237
 238 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring,
 239                                            void *elem);
 240 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring,
 241                                            void *elem);
 242 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem);
 243 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem);
 244
 245 static struct k3_nav_ring_ops k3_nav_mode_proxy_ops = {
 246                 .push_tail = k3_ringacc_ring_push_tail_proxy,
 247                 .push_head = k3_ringacc_ring_push_head_proxy,
 248                 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
 249                 .pop_head = k3_ringacc_ring_pop_head_proxy,
 250 };
 251
 252 struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
 253 {
 254         return ringacc->dev;
 255 }
 256
 257 struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
 258                                                 int id, u32 flags)
 259 {
 260         int proxy_id = K3_RINGACC_PROXY_NOT_USED;
 261
 262         if (id == K3_NAV_RINGACC_RING_ID_ANY) {
 263                 /* Request for any general purpose ring */
 264                 struct ti_sci_resource_desc *gp_rings =
 265                                         &ringacc->rm_gp_range->desc[0];
 266                 unsigned long size;
 267
 268                 size = gp_rings->start + gp_rings->num;
 269                 id = find_next_zero_bit(ringacc->rings_inuse,
 270                                         size, gp_rings->start);
 271                 if (id == size)
 272                         goto error;
 273         } else if (id < 0) {
 274                 goto error;
 275         }
 276
 277         if (test_bit(id, ringacc->rings_inuse) &&
 278             !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
 279                 goto error;
 280         else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
 281                 goto out;
 282
 283         if (flags & K3_NAV_RINGACC_RING_USE_PROXY) {
 284                 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
 285                                               ringacc->num_proxies, 0);
 286                 if (proxy_id == ringacc->num_proxies)
 287                         goto error;
 288         }
 289
 290         if (!try_module_get(ringacc->dev->driver->owner))
 291                 goto error;
 292
 293         if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 294                 set_bit(proxy_id, ringacc->proxy_inuse);
 295                 ringacc->rings[id].proxy_id = proxy_id;
 296                 pr_debug("Giving ring#%d proxy#%d\n",
 297                          id, proxy_id);
 298         } else {
 299                 pr_debug("Giving ring#%d\n", id);
 300         }
 301
 302         set_bit(id, ringacc->rings_inuse);
 303 out:
 304         ringacc->rings[id].use_count++;
 305         return &ringacc->rings[id];
 306
 307 error:
 308         return NULL;
 309 }
 310
 311 static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
 312 {
 313         struct k3_nav_ringacc *ringacc = ring->parent;
 314         int ret;
 315
 316         ret = ringacc->tisci_ring_ops->config(
 317                         ringacc->tisci,
 318                         TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
 319                         ringacc->tisci_dev_id,
 320                         ring->ring_id,
 321                         0,
 322                         0,
 323                         ring->size,
 324                         0,
 325                         0,
 326                         0);
 327         if (ret)
 328                 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
 329                         ret, ring->ring_id);
 330 }
 331
 332 void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
 333 {
 334         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 335                 return;
 336
 337         ring->occ = 0;
 338         ring->free = 0;
 339         ring->rindex = 0;
 340         ring->windex = 0;
 341
 342         k3_ringacc_ring_reset_sci(ring);
 343 }
 344
 345 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
 346                                                enum k3_nav_ring_mode mode)
 347 {
 348         struct k3_nav_ringacc *ringacc = ring->parent;
 349         int ret;
 350
 351         ret = ringacc->tisci_ring_ops->config(
 352                         ringacc->tisci,
 353                         TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
 354                         ringacc->tisci_dev_id,
 355                         ring->ring_id,
 356                         0,
 357                         0,
 358                         0,
 359                         mode,
 360                         0,
 361                         0);
 362         if (ret)
 363                 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
 364                         ret, ring->ring_id);
 365 }
 366
 367 void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
 368 {
 369         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 370                 return;
 371
 372         if (!ring->parent->dma_ring_reset_quirk) {
 373                 k3_nav_ringacc_ring_reset(ring);
 374                 return;
 375         }
 376
 377         if (!occ)
 378                 occ = ringacc_readl(&ring->rt->occ);
 379
 380         if (occ) {
 381                 u32 db_ring_cnt, db_ring_cnt_cur;
 382
 383                 pr_debug("%s %u occ: %u\n", __func__,
 384                          ring->ring_id, occ);
 385                 /* 2. Reset the ring */
 386                 k3_ringacc_ring_reset_sci(ring);
 387
 388                 /*
 389                  * 3. Setup the ring in ring/doorbell mode
 390                  * (if not already in this mode)
 391                  */
 392                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 393                         k3_ringacc_ring_reconfig_qmode_sci(
 394                                         ring, K3_NAV_RINGACC_RING_MODE_RING);
 395                 /*
 396                  * 4. Ring the doorbell 2**22 – ringOcc times.
 397                  * This will wrap the internal UDMAP ring state occupancy
 398                  * counter (which is 21-bits wide) to 0.
 399                  */
 400                 db_ring_cnt = (1U << 22) - occ;
 401
 402                 while (db_ring_cnt != 0) {
 403                         /*
 404                          * Ring the doorbell with the maximum count each
 405                          * iteration if possible to minimize the total
 406                          * of writes
 407                          */
 408                         if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
 409                                 db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
 410                         else
 411                                 db_ring_cnt_cur = db_ring_cnt;
 412
 413                         writel(db_ring_cnt_cur, &ring->rt->db);
 414                         db_ring_cnt -= db_ring_cnt_cur;
 415                 }
 416
 417                 /* 5. Restore the original ring mode (if not ring mode) */
 418                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 419                         k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
 420         }
 421
 422         /* 2. Reset the ring */
 423         k3_nav_ringacc_ring_reset(ring);
 424 }
 425
 426 static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
 427 {
 428         struct k3_nav_ringacc *ringacc = ring->parent;
 429         int ret;
 430
 431         ret = ringacc->tisci_ring_ops->config(
 432                         ringacc->tisci,
 433                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 434                         ringacc->tisci_dev_id,
 435                         ring->ring_id,
 436                         0,
 437                         0,
 438                         0,
 439                         0,
 440                         0,
 441                         0);
 442         if (ret)
 443                 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
 444                         ret, ring->ring_id);
 445 }
 446
 447 int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
 448 {
 449         struct k3_nav_ringacc *ringacc;
 450
 451         if (!ring)
 452                 return -EINVAL;
 453
 454         ringacc = ring->parent;
 455
 456         pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
 457
 458         if (!test_bit(ring->ring_id, ringacc->rings_inuse))
 459                 return -EINVAL;
 460
 461         if (--ring->use_count)
 462                 goto out;
 463
 464         if (!(ring->flags & KNAV_RING_FLAG_BUSY))
 465                 goto no_init;
 466
 467         k3_ringacc_ring_free_sci(ring);
 468
 469         dma_free_coherent(ringacc->dev,
 470                           ring->size * (4 << ring->elm_size),
 471                           ring->ring_mem_virt, ring->ring_mem_dma);
 472         ring->flags &= ~KNAV_RING_FLAG_BUSY;
 473         ring->ops = NULL;
 474         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 475                 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
 476                 ring->proxy = NULL;
 477                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
 478         }
 479
 480 no_init:
 481         clear_bit(ring->ring_id, ringacc->rings_inuse);
 482
 483         module_put(ringacc->dev->driver->owner);
 484
 485 out:
 486         return 0;
 487 }
 488
 489 u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
 490 {
 491         if (!ring)
 492                 return -EINVAL;
 493
 494         return ring->ring_id;
 495 }
 496
 497 static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
 498 {
 499         struct k3_nav_ringacc *ringacc = ring->parent;
 500         u32 ring_idx;
 501         int ret;
 502
 503         if (!ringacc->tisci)
 504                 return -EINVAL;
 505
 506         ring_idx = ring->ring_id;
 507         ret = ringacc->tisci_ring_ops->config(
 508                         ringacc->tisci,
 509                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 510                         ringacc->tisci_dev_id,
 511                         ring_idx,
 512                         lower_32_bits(ring->ring_mem_dma),
 513                         upper_32_bits(ring->ring_mem_dma),
 514                         ring->size,
 515                         ring->mode,
 516                         ring->elm_size,
 517                         0);
 518         if (ret)
 519                 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
 520                         ret, ring_idx);
 521
 522         return ret;
 523 }
 524
 525 int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
 526                             struct k3_nav_ring_cfg *cfg)
 527 {
 528         struct k3_nav_ringacc *ringacc = ring->parent;
 529         int ret = 0;
 530
 531         if (!ring || !cfg)
 532                 return -EINVAL;
 533         if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
 534             cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
 535             cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
 536             !test_bit(ring->ring_id, ringacc->rings_inuse))
 537                 return -EINVAL;
 538
 539         if (ring->use_count != 1)
 540                 return 0;
 541
 542         ring->size = cfg->size;
 543         ring->elm_size = cfg->elm_size;
 544         ring->mode = cfg->mode;
 545         ring->occ = 0;
 546         ring->free = 0;
 547         ring->rindex = 0;
 548         ring->windex = 0;
 549
 550         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
 551                 ring->proxy = ringacc->proxy_target_base +
 552                               ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
 553
 554         switch (ring->mode) {
 555         case K3_NAV_RINGACC_RING_MODE_RING:
 556                 ring->ops = &k3_nav_mode_ring_ops;
 557                 break;
 558         case K3_NAV_RINGACC_RING_MODE_QM:
 559                 /*
 560                  * In Queue mode elm_size can be 8 only and each operation
 561                  * uses 2 element slots
 562                  */
 563                 if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
 564                     cfg->size % 2)
 565                         goto err_free_proxy;
 566         case K3_NAV_RINGACC_RING_MODE_MESSAGE:
 567                 if (ring->proxy)
 568                         ring->ops = &k3_nav_mode_proxy_ops;
 569                 else
 570                         ring->ops = &k3_nav_mode_msg_ops;
 571                 break;
 572         default:
 573                 ring->ops = NULL;
 574                 ret = -EINVAL;
 575                 goto err_free_proxy;
 576         };
 577
 578         ring->ring_mem_virt =
 579                         dma_zalloc_coherent(ringacc->dev,
 580                                             ring->size * (4 << ring->elm_size),
 581                                             &ring->ring_mem_dma, GFP_KERNEL);
 582         if (!ring->ring_mem_virt) {
 583                 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
 584                 ret = -ENOMEM;
 585                 goto err_free_ops;
 586         }
 587
 588         ret = k3_nav_ringacc_ring_cfg_sci(ring);
 589
 590         if (ret)
 591                 goto err_free_mem;
 592
 593         ring->flags |= KNAV_RING_FLAG_BUSY;
 594         ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
 595                         K3_NAV_RING_FLAG_SHARED : 0;
 596
 597         return 0;
 598
 599 err_free_mem:
 600         dma_free_coherent(ringacc->dev,
 601                           ring->size * (4 << ring->elm_size),
 602                           ring->ring_mem_virt,
 603                           ring->ring_mem_dma);
 604 err_free_ops:
 605         ring->ops = NULL;
 606 err_free_proxy:
 607         ring->proxy = NULL;
 608         return ret;
 609 }
 610
 611 u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
 612 {
 613         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 614                 return -EINVAL;
 615
 616         return ring->size;
 617 }
 618
 619 u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
 620 {
 621         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 622                 return -EINVAL;
 623
 624         if (!ring->free)
 625                 ring->free = ring->size - ringacc_readl(&ring->rt->occ);
 626
 627         return ring->free;
 628 }
 629
 630 u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
 631 {
 632         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 633                 return -EINVAL;
 634
 635         return ringacc_readl(&ring->rt->occ);
 636 }
 637
 638 u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
 639 {
 640         return !k3_nav_ringacc_ring_get_free(ring);
 641 }
 642
 643 enum k3_ringacc_access_mode {
 644         K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
 645         K3_RINGACC_ACCESS_MODE_POP_HEAD,
 646         K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
 647         K3_RINGACC_ACCESS_MODE_POP_TAIL,
 648         K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
 649         K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
 650 };
 651
 652 static int k3_ringacc_ring_cfg_proxy(struct k3_nav_ring *ring,
 653                                      enum k3_ringacc_proxy_access_mode mode)
 654 {
 655         u32 val;
 656
 657         val = ring->ring_id;
 658         val |= mode << 16;
 659         val |= ring->elm_size << 24;
 660         ringacc_writel(val, &ring->proxy->control);
 661         return 0;
 662 }
 663
 664 static int k3_nav_ringacc_ring_access_proxy(
 665                         struct k3_nav_ring *ring, void *elem,
 666                         enum k3_ringacc_access_mode access_mode)
 667 {
 668         void __iomem *ptr;
 669
 670         ptr = (void __iomem *)&ring->proxy->data;
 671
 672         switch (access_mode) {
 673         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 674         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 675                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
 676                 break;
 677         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 678         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 679                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
 680                 break;
 681         default:
 682                 return -EINVAL;
 683         }
 684
 685         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 686
 687         switch (access_mode) {
 688         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 689         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 690                 pr_debug("proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 691                          ptr, access_mode);
 692                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 693                 ring->occ--;
 694                 break;
 695         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 696         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 697                 pr_debug("proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
 698                          ptr, access_mode);
 699                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 700                 ring->free--;
 701                 break;
 702         default:
 703                 return -EINVAL;
 704         }
 705
 706         pr_debug("proxy: free%d occ%d\n",
 707                  ring->free, ring->occ);
 708         return 0;
 709 }
 710
 711 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring, void *elem)
 712 {
 713         return k3_nav_ringacc_ring_access_proxy(
 714                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 715 }
 716
 717 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring, void *elem)
 718 {
 719         return k3_nav_ringacc_ring_access_proxy(
 720                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 721 }
 722
 723 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem)
 724 {
 725         return k3_nav_ringacc_ring_access_proxy(
 726                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 727 }
 728
 729 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem)
 730 {
 731         return k3_nav_ringacc_ring_access_proxy(
 732                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 733 }
 734
 735 static int k3_nav_ringacc_ring_access_io(
 736                 struct k3_nav_ring *ring, void *elem,
 737                 enum k3_ringacc_access_mode access_mode)
 738 {
 739         void __iomem *ptr;
 740
 741         switch (access_mode) {
 742         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 743         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 744                 ptr = (void __iomem *)&ring->fifos->head_data;
 745                 break;
 746         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 747         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 748                 ptr = (void __iomem *)&ring->fifos->tail_data;
 749                 break;
 750         default:
 751                 return -EINVAL;
 752         }
 753
 754         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 755
 756         switch (access_mode) {
 757         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 758         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 759                 pr_debug("memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 760                          ptr, access_mode);
 761                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 762                 ring->occ--;
 763                 break;
 764         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 765         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 766                 pr_debug("memcpy_toio(x): --> ptr(%p), mode:%d\n",
 767                          ptr, access_mode);
 768                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 769                 ring->free--;
 770                 break;
 771         default:
 772                 return -EINVAL;
 773         }
 774
 775         pr_debug("free%d index%d occ%d index%d\n",
 776                  ring->free, ring->windex, ring->occ, ring->rindex);
 777         return 0;
 778 }
 779
 780 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 781                                             void *elem)
 782 {
 783         return k3_nav_ringacc_ring_access_io(
 784                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 785 }
 786
 787 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem)
 788 {
 789         return k3_nav_ringacc_ring_access_io(
 790                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 791 }
 792
 793 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem)
 794 {
 795         return k3_nav_ringacc_ring_access_io(
 796                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 797 }
 798
 799 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring, void *elem)
 800 {
 801         return k3_nav_ringacc_ring_access_io(
 802                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 803 }
 804
 805 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
 806 {
 807         void *elem_ptr;
 808
 809         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->windex);
 810
 811         memcpy(elem_ptr, elem, (4 << ring->elm_size));
 812
 813         flush_dcache_range((unsigned long)ring->ring_mem_virt,
 814                            ALIGN((unsigned long)ring->ring_mem_virt +
 815                                  ring->size * (4 << ring->elm_size),
 816                                  ARCH_DMA_MINALIGN));
 817
 818         ring->windex = (ring->windex + 1) % ring->size;
 819         ring->free--;
 820         ringacc_writel(1, &ring->rt->db);
 821
 822         pr_debug("ring_push_mem: free%d index%d\n",
 823                  ring->free, ring->windex);
 824
 825         return 0;
 826 }
 827
 828 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
 829 {
 830         void *elem_ptr;
 831
 832         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->rindex);
 833
 834         invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
 835                                 ALIGN((unsigned long)ring->ring_mem_virt +
 836                                       ring->size * (4 << ring->elm_size),
 837                                       ARCH_DMA_MINALIGN));
 838
 839         memcpy(elem, elem_ptr, (4 << ring->elm_size));
 840
 841         ring->rindex = (ring->rindex + 1) % ring->size;
 842         ring->occ--;
 843         ringacc_writel(-1, &ring->rt->db);
 844
 845         pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
 846                  ring->occ, ring->rindex, elem_ptr);
 847         return 0;
 848 }
 849
 850 int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
 851 {
 852         int ret = -EOPNOTSUPP;
 853
 854         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 855                 return -EINVAL;
 856
 857         pr_debug("ring_push%d: free%d index%d\n",
 858                  ring->ring_id, ring->free, ring->windex);
 859
 860         if (k3_nav_ringacc_ring_is_full(ring))
 861                 return -ENOMEM;
 862
 863         if (ring->ops && ring->ops->push_tail)
 864                 ret = ring->ops->push_tail(ring, elem);
 865
 866         return ret;
 867 }
 868
 869 int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
 870 {
 871         int ret = -EOPNOTSUPP;
 872
 873         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 874                 return -EINVAL;
 875
 876         pr_debug("ring_push_head: free%d index%d\n",
 877                  ring->free, ring->windex);
 878
 879         if (k3_nav_ringacc_ring_is_full(ring))
 880                 return -ENOMEM;
 881
 882         if (ring->ops && ring->ops->push_head)
 883                 ret = ring->ops->push_head(ring, elem);
 884
 885         return ret;
 886 }
 887
 888 int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
 889 {
 890         int ret = -EOPNOTSUPP;
 891
 892         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 893                 return -EINVAL;
 894
 895         if (!ring->occ)
 896                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 897
 898         pr_debug("ring_pop%d: occ%d index%d\n",
 899                  ring->ring_id, ring->occ, ring->rindex);
 900
 901         if (!ring->occ)
 902                 return -ENODATA;
 903
 904         if (ring->ops && ring->ops->pop_head)
 905                 ret = ring->ops->pop_head(ring, elem);
 906
 907         return ret;
 908 }
 909
 910 int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
 911 {
 912         int ret = -EOPNOTSUPP;
 913
 914         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 915                 return -EINVAL;
 916
 917         if (!ring->occ)
 918                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 919
 920         pr_debug("ring_pop_tail: occ%d index%d\n",
 921                  ring->occ, ring->rindex);
 922
 923         if (!ring->occ)
 924                 return -ENODATA;
 925
 926         if (ring->ops && ring->ops->pop_tail)
 927                 ret = ring->ops->pop_tail(ring, elem);
 928
 929         return ret;
 930 }
 931
 932 static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
 933 {
 934         struct udevice *dev = ringacc->dev;
 935         struct udevice *tisci_dev = NULL;
 936         int ret;
 937
 938         ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
 939         if (!ringacc->num_rings) {
 940                 dev_err(dev, "ti,num-rings read failure %d\n", ret);
 941                 return -EINVAL;
 942         }
 943
 944         ringacc->dma_ring_reset_quirk =
 945                         dev_read_bool(dev, "ti,dma-ring-reset-quirk");
 946
 947         ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, dev,
 948                                            "ti,sci", &tisci_dev);
 949         if (ret) {
 950                 pr_debug("TISCI RA RM get failed (%d)\n", ret);
 951                 ringacc->tisci = NULL;
 952                 return -ENODEV;
 953         }
 954         ringacc->tisci = (struct ti_sci_handle *)
 955                          (ti_sci_get_handle_from_sysfw(tisci_dev));
 956
 957         ret = dev_read_u32_default(dev, "ti,sci", 0);
 958         if (!ret) {
 959                 dev_err(dev, "TISCI RA RM disabled\n");
 960                 ringacc->tisci = NULL;
 961                 return ret;
 962         }
 963
 964         ret = dev_read_u32(dev, "ti,sci-dev-id", &ringacc->tisci_dev_id);
 965         if (ret) {
 966                 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
 967                 ringacc->tisci = NULL;
 968                 return ret;
 969         }
 970
 971         ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
 972                                         ringacc->tisci, dev,
 973                                         ringacc->tisci_dev_id,
 974                                         "ti,sci-rm-range-gp-rings");
 975         if (IS_ERR(ringacc->rm_gp_range))
 976                 ret = PTR_ERR(ringacc->rm_gp_range);
 977
 978         return 0;
 979 }
 980
 981 static int k3_nav_ringacc_probe(struct udevice *dev)
 982 {
 983         struct k3_nav_ringacc *ringacc;
 984         void __iomem *base_fifo, *base_rt;
 985         int ret, i;
 986
 987         ringacc = dev_get_priv(dev);
 988         if (!ringacc)
 989                 return -ENOMEM;
 990
 991         ringacc->dev = dev;
 992
 993         ret = k3_nav_ringacc_probe_dt(ringacc);
 994         if (ret)
 995                 return ret;
 996
 997         base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
 998         pr_debug("rt %p\n", base_rt);
 999         if (IS_ERR(base_rt))
1000                 return PTR_ERR(base_rt);
1001
1002         base_fifo = (uint32_t *)devfdt_get_addr_name(dev, "fifos");
1003         pr_debug("fifos %p\n", base_fifo);
1004         if (IS_ERR(base_fifo))
1005                 return PTR_ERR(base_fifo);
1006
1007         ringacc->proxy_gcfg = (struct k3_ringacc_proxy_gcfg_regs __iomem *)
1008                 devfdt_get_addr_name(dev, "proxy_gcfg");
1009         if (IS_ERR(ringacc->proxy_gcfg))
1010                 return PTR_ERR(ringacc->proxy_gcfg);
1011         ringacc->proxy_target_base =
1012                 (struct k3_ringacc_proxy_gcfg_regs __iomem *)
1013                 devfdt_get_addr_name(dev, "proxy_target");
1014         if (IS_ERR(ringacc->proxy_target_base))
1015                 return PTR_ERR(ringacc->proxy_target_base);
1016
1017         ringacc->num_proxies = ringacc_readl(&ringacc->proxy_gcfg->config) &
1018                                          K3_RINGACC_PROXY_CFG_THREADS_MASK;
1019
1020         ringacc->rings = devm_kzalloc(dev,
1021                                       sizeof(*ringacc->rings) *
1022                                       ringacc->num_rings,
1023                                       GFP_KERNEL);
1024         ringacc->rings_inuse = devm_kcalloc(dev,
1025                                             BITS_TO_LONGS(ringacc->num_rings),
1026                                             sizeof(unsigned long), GFP_KERNEL);
1027         ringacc->proxy_inuse = devm_kcalloc(dev,
1028                                             BITS_TO_LONGS(ringacc->num_proxies),
1029                                             sizeof(unsigned long), GFP_KERNEL);
1030
1031         if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1032                 return -ENOMEM;
1033
1034         for (i = 0; i < ringacc->num_rings; i++) {
1035                 ringacc->rings[i].rt = base_rt +
1036                                        KNAV_RINGACC_RT_REGS_STEP * i;
1037                 ringacc->rings[i].fifos = base_fifo +
1038                                           KNAV_RINGACC_FIFO_REGS_STEP * i;
1039                 ringacc->rings[i].parent = ringacc;
1040                 ringacc->rings[i].ring_id = i;
1041                 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1042         }
1043         dev_set_drvdata(dev, ringacc);
1044
1045         ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1046
1047         list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
1048
1049         dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1050                  ringacc->num_rings,
1051                  ringacc->rm_gp_range->desc[0].start,
1052                  ringacc->rm_gp_range->desc[0].num,
1053                  ringacc->tisci_dev_id);
1054         dev_info(dev, "dma-ring-reset-quirk: %s\n",
1055                  ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1056         dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1057                  ringacc_readl(&ringacc->proxy_gcfg->revision),
1058                  ringacc->num_proxies);
1059         return 0;
1060 }
1061
1062 static const struct udevice_id knav_ringacc_ids[] = {
1063         { .compatible = "ti,am654-navss-ringacc" },
1064         {},
1065 };
1066
1067 U_BOOT_DRIVER(k3_navss_ringacc) = {
1068         .name   = "k3-navss-ringacc",
1069         .id     = UCLASS_MISC,
1070         .of_match = knav_ringacc_ids,
1071         .probe = k3_nav_ringacc_probe,
1072         .priv_auto_alloc_size = sizeof(struct k3_nav_ringacc),
1073 };