drivers/soc/ti/k3-navss-ringacc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
   4  *
   5  * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
   6  */
   7
   8 #include <common.h>
   9 #include <asm/io.h>
  10 #include <malloc.h>
  11 #include <asm/dma-mapping.h>
  12 #include <asm/bitops.h>
  13 #include <dm.h>
  14 #include <dm/read.h>
  15 #include <dm/uclass.h>
  16 #include <linux/compat.h>
  17 #include <linux/soc/ti/k3-navss-ringacc.h>
  18 #include <linux/soc/ti/ti_sci_protocol.h>
  19
  20 #define set_bit(bit, bitmap)    __set_bit(bit, bitmap)
  21 #define clear_bit(bit, bitmap)  __clear_bit(bit, bitmap)
  22 #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
  23         dma_free_coherent(cpu_addr)
  24 #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
  25 ({ \
  26         void    *ring_mem_virt; \
  27         ring_mem_virt = dma_alloc_coherent((size), \
  28                                            (unsigned long *)(dma_handle)); \
  29         if (ring_mem_virt) \
  30                 memset(ring_mem_virt, 0, (size)); \
  31         ring_mem_virt; \
  32 })
  33
  34 static LIST_HEAD(k3_nav_ringacc_list);
  35
  36 static  void ringacc_writel(u32 v, void __iomem *reg)
  37 {
  38         pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
  39         writel(v, reg);
  40 }
  41
  42 static  u32 ringacc_readl(void __iomem *reg)
  43 {
  44         u32 v;
  45
  46         v = readl(reg);
  47         pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
  48         return v;
  49 }
  50
  51 #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK           GENMASK(19, 0)
  52
  53 /**
  54  * struct k3_nav_ring_rt_regs -  The RA Control/Status Registers region
  55  */
  56 struct k3_nav_ring_rt_regs {
  57         u32     resv_16[4];
  58         u32     db;             /* RT Ring N Doorbell Register */
  59         u32     resv_4[1];
  60         u32     occ;            /* RT Ring N Occupancy Register */
  61         u32     indx;           /* RT Ring N Current Index Register */
  62         u32     hwocc;          /* RT Ring N Hardware Occupancy Register */
  63         u32     hwindx;         /* RT Ring N Current Index Register */
  64 };
  65
  66 #define KNAV_RINGACC_RT_REGS_STEP       0x1000
  67
  68 /**
  69  * struct k3_nav_ring_fifo_regs -  The Ring Accelerator Queues Registers region
  70  */
  71 struct k3_nav_ring_fifo_regs {
  72         u32     head_data[128];         /* Ring Head Entry Data Registers */
  73         u32     tail_data[128];         /* Ring Tail Entry Data Registers */
  74         u32     peek_head_data[128];    /* Ring Peek Head Entry Data Regs */
  75         u32     peek_tail_data[128];    /* Ring Peek Tail Entry Data Regs */
  76 };
  77
  78 /**
  79  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
  80  */
  81 struct k3_ringacc_proxy_gcfg_regs {
  82         u32     revision;       /* Revision Register */
  83         u32     config;         /* Config Register */
  84 };
  85
  86 #define K3_RINGACC_PROXY_CFG_THREADS_MASK               GENMASK(15, 0)
  87
  88 /**
  89  * struct k3_ringacc_proxy_target_regs -  RA Proxy Datapath MMIO Region
  90  */
  91 struct k3_ringacc_proxy_target_regs {
  92         u32     control;        /* Proxy Control Register */
  93         u32     status;         /* Proxy Status Register */
  94         u8      resv_512[504];
  95         u32     data[128];      /* Proxy Data Register */
  96 };
  97
  98 #define K3_RINGACC_PROXY_TARGET_STEP    0x1000
  99 #define K3_RINGACC_PROXY_NOT_USED       (-1)
 100
 101 enum k3_ringacc_proxy_access_mode {
 102         PROXY_ACCESS_MODE_HEAD = 0,
 103         PROXY_ACCESS_MODE_TAIL = 1,
 104         PROXY_ACCESS_MODE_PEEK_HEAD = 2,
 105         PROXY_ACCESS_MODE_PEEK_TAIL = 3,
 106 };
 107
 108 #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
 109 #define KNAV_RINGACC_FIFO_REGS_STEP     0x1000
 110 #define KNAV_RINGACC_MAX_DB_RING_CNT    (127U)
 111
 112 /**
 113  * struct k3_nav_ring_ops -  Ring operations
 114  */
 115 struct k3_nav_ring_ops {
 116         int (*push_tail)(struct k3_nav_ring *ring, void *elm);
 117         int (*push_head)(struct k3_nav_ring *ring, void *elm);
 118         int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
 119         int (*pop_head)(struct k3_nav_ring *ring, void *elm);
 120 };
 121
 122 /**
 123  * struct k3_nav_ring - RA Ring descriptor
 124  *
 125  * @rt - Ring control/status registers
 126  * @fifos - Ring queues registers
 127  * @proxy - Ring Proxy Datapath registers
 128  * @ring_mem_dma - Ring buffer dma address
 129  * @ring_mem_virt - Ring buffer virt address
 130  * @ops - Ring operations
 131  * @size - Ring size in elements
 132  * @elm_size - Size of the ring element
 133  * @mode - Ring mode
 134  * @flags - flags
 135  * @free - Number of free elements
 136  * @occ - Ring occupancy
 137  * @windex - Write index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 138  * @rindex - Read index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 139  * @ring_id - Ring Id
 140  * @parent - Pointer on struct @k3_nav_ringacc
 141  * @use_count - Use count for shared rings
 142  * @proxy_id - RA Ring Proxy Id (only if @K3_NAV_RINGACC_RING_USE_PROXY)
 143  */
 144 struct k3_nav_ring {
 145         struct k3_nav_ring_rt_regs __iomem *rt;
 146         struct k3_nav_ring_fifo_regs __iomem *fifos;
 147         struct k3_ringacc_proxy_target_regs  __iomem *proxy;
 148         dma_addr_t      ring_mem_dma;
 149         void            *ring_mem_virt;
 150         struct k3_nav_ring_ops *ops;
 151         u32             size;
 152         enum k3_nav_ring_size elm_size;
 153         enum k3_nav_ring_mode mode;
 154         u32             flags;
 155 #define KNAV_RING_FLAG_BUSY     BIT(1)
 156 #define K3_NAV_RING_FLAG_SHARED BIT(2)
 157         u32             free;
 158         u32             occ;
 159         u32             windex;
 160         u32             rindex;
 161         u32             ring_id;
 162         struct k3_nav_ringacc   *parent;
 163         u32             use_count;
 164         int             proxy_id;
 165 };
 166
 167 /**
 168  * struct k3_nav_ringacc - Rings accelerator descriptor
 169  *
 170  * @dev - pointer on RA device
 171  * @proxy_gcfg - RA proxy global config registers
 172  * @proxy_target_base - RA proxy datapath region
 173  * @num_rings - number of ring in RA
 174  * @rm_gp_range - general purpose rings range from tisci
 175  * @dma_ring_reset_quirk - DMA reset w/a enable
 176  * @num_proxies - number of RA proxies
 177  * @rings - array of rings descriptors (struct @k3_nav_ring)
 178  * @list - list of RAs in the system
 179  * @tisci - pointer ti-sci handle
 180  * @tisci_ring_ops - ti-sci rings ops
 181  * @tisci_dev_id - ti-sci device id
 182  */
 183 struct k3_nav_ringacc {
 184         struct udevice *dev;
 185         struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
 186         void __iomem *proxy_target_base;
 187         u32 num_rings; /* number of rings in Ringacc module */
 188         unsigned long *rings_inuse;
 189         struct ti_sci_resource *rm_gp_range;
 190         bool dma_ring_reset_quirk;
 191         u32 num_proxies;
 192         unsigned long *proxy_inuse;
 193
 194         struct k3_nav_ring *rings;
 195         struct list_head list;
 196
 197         const struct ti_sci_handle *tisci;
 198         const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
 199         u32  tisci_dev_id;
 200 };
 201
 202 static long k3_nav_ringacc_ring_get_fifo_pos(struct k3_nav_ring *ring)
 203 {
 204         return KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES -
 205                (4 << ring->elm_size);
 206 }
 207
 208 static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
 209 {
 210         return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
 211 }
 212
 213 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
 214 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
 215
 216 static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
 217                 .push_tail = k3_nav_ringacc_ring_push_mem,
 218                 .pop_head = k3_nav_ringacc_ring_pop_mem,
 219 };
 220
 221 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem);
 222 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem);
 223 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 224                                             void *elem);
 225 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring,
 226                                            void *elem);
 227
 228 static struct k3_nav_ring_ops k3_nav_mode_msg_ops = {
 229                 .push_tail = k3_nav_ringacc_ring_push_io,
 230                 .push_head = k3_nav_ringacc_ring_push_head_io,
 231                 .pop_tail = k3_nav_ringacc_ring_pop_tail_io,
 232                 .pop_head = k3_nav_ringacc_ring_pop_io,
 233 };
 234
 235 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring,
 236                                            void *elem);
 237 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring,
 238                                            void *elem);
 239 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem);
 240 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem);
 241
 242 static struct k3_nav_ring_ops k3_nav_mode_proxy_ops = {
 243                 .push_tail = k3_ringacc_ring_push_tail_proxy,
 244                 .push_head = k3_ringacc_ring_push_head_proxy,
 245                 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
 246                 .pop_head = k3_ringacc_ring_pop_head_proxy,
 247 };
 248
 249 struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
 250 {
 251         return ringacc->dev;
 252 }
 253
 254 struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
 255                                                 int id, u32 flags)
 256 {
 257         int proxy_id = K3_RINGACC_PROXY_NOT_USED;
 258
 259         if (id == K3_NAV_RINGACC_RING_ID_ANY) {
 260                 /* Request for any general purpose ring */
 261                 struct ti_sci_resource_desc *gp_rings =
 262                                         &ringacc->rm_gp_range->desc[0];
 263                 unsigned long size;
 264
 265                 size = gp_rings->start + gp_rings->num;
 266                 id = find_next_zero_bit(ringacc->rings_inuse,
 267                                         size, gp_rings->start);
 268                 if (id == size)
 269                         goto error;
 270         } else if (id < 0) {
 271                 goto error;
 272         }
 273
 274         if (test_bit(id, ringacc->rings_inuse) &&
 275             !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
 276                 goto error;
 277         else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
 278                 goto out;
 279
 280         if (flags & K3_NAV_RINGACC_RING_USE_PROXY) {
 281                 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
 282                                               ringacc->num_proxies, 0);
 283                 if (proxy_id == ringacc->num_proxies)
 284                         goto error;
 285         }
 286
 287         if (!try_module_get(ringacc->dev->driver->owner))
 288                 goto error;
 289
 290         if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 291                 set_bit(proxy_id, ringacc->proxy_inuse);
 292                 ringacc->rings[id].proxy_id = proxy_id;
 293                 pr_debug("Giving ring#%d proxy#%d\n",
 294                          id, proxy_id);
 295         } else {
 296                 pr_debug("Giving ring#%d\n", id);
 297         }
 298
 299         set_bit(id, ringacc->rings_inuse);
 300 out:
 301         ringacc->rings[id].use_count++;
 302         return &ringacc->rings[id];
 303
 304 error:
 305         return NULL;
 306 }
 307
 308 static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
 309 {
 310         struct k3_nav_ringacc *ringacc = ring->parent;
 311         int ret;
 312
 313         ret = ringacc->tisci_ring_ops->config(
 314                         ringacc->tisci,
 315                         TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
 316                         ringacc->tisci_dev_id,
 317                         ring->ring_id,
 318                         0,
 319                         0,
 320                         ring->size,
 321                         0,
 322                         0,
 323                         0);
 324         if (ret)
 325                 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
 326                         ret, ring->ring_id);
 327 }
 328
 329 void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
 330 {
 331         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 332                 return;
 333
 334         ring->occ = 0;
 335         ring->free = 0;
 336         ring->rindex = 0;
 337         ring->windex = 0;
 338
 339         k3_ringacc_ring_reset_sci(ring);
 340 }
 341
 342 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
 343                                                enum k3_nav_ring_mode mode)
 344 {
 345         struct k3_nav_ringacc *ringacc = ring->parent;
 346         int ret;
 347
 348         ret = ringacc->tisci_ring_ops->config(
 349                         ringacc->tisci,
 350                         TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
 351                         ringacc->tisci_dev_id,
 352                         ring->ring_id,
 353                         0,
 354                         0,
 355                         0,
 356                         mode,
 357                         0,
 358                         0);
 359         if (ret)
 360                 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
 361                         ret, ring->ring_id);
 362 }
 363
 364 void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
 365 {
 366         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 367                 return;
 368
 369         if (!ring->parent->dma_ring_reset_quirk) {
 370                 k3_nav_ringacc_ring_reset(ring);
 371                 return;
 372         }
 373
 374         if (!occ)
 375                 occ = ringacc_readl(&ring->rt->occ);
 376
 377         if (occ) {
 378                 u32 db_ring_cnt, db_ring_cnt_cur;
 379
 380                 pr_debug("%s %u occ: %u\n", __func__,
 381                          ring->ring_id, occ);
 382                 /* 2. Reset the ring */
 383                 k3_ringacc_ring_reset_sci(ring);
 384
 385                 /*
 386                  * 3. Setup the ring in ring/doorbell mode
 387                  * (if not already in this mode)
 388                  */
 389                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 390                         k3_ringacc_ring_reconfig_qmode_sci(
 391                                         ring, K3_NAV_RINGACC_RING_MODE_RING);
 392                 /*
 393                  * 4. Ring the doorbell 2**22 – ringOcc times.
 394                  * This will wrap the internal UDMAP ring state occupancy
 395                  * counter (which is 21-bits wide) to 0.
 396                  */
 397                 db_ring_cnt = (1U << 22) - occ;
 398
 399                 while (db_ring_cnt != 0) {
 400                         /*
 401                          * Ring the doorbell with the maximum count each
 402                          * iteration if possible to minimize the total
 403                          * of writes
 404                          */
 405                         if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
 406                                 db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
 407                         else
 408                                 db_ring_cnt_cur = db_ring_cnt;
 409
 410                         writel(db_ring_cnt_cur, &ring->rt->db);
 411                         db_ring_cnt -= db_ring_cnt_cur;
 412                 }
 413
 414                 /* 5. Restore the original ring mode (if not ring mode) */
 415                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 416                         k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
 417         }
 418
 419         /* 2. Reset the ring */
 420         k3_nav_ringacc_ring_reset(ring);
 421 }
 422
 423 static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
 424 {
 425         struct k3_nav_ringacc *ringacc = ring->parent;
 426         int ret;
 427
 428         ret = ringacc->tisci_ring_ops->config(
 429                         ringacc->tisci,
 430                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 431                         ringacc->tisci_dev_id,
 432                         ring->ring_id,
 433                         0,
 434                         0,
 435                         0,
 436                         0,
 437                         0,
 438                         0);
 439         if (ret)
 440                 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
 441                         ret, ring->ring_id);
 442 }
 443
 444 int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
 445 {
 446         struct k3_nav_ringacc *ringacc;
 447
 448         if (!ring)
 449                 return -EINVAL;
 450
 451         ringacc = ring->parent;
 452
 453         pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
 454
 455         if (!test_bit(ring->ring_id, ringacc->rings_inuse))
 456                 return -EINVAL;
 457
 458         if (--ring->use_count)
 459                 goto out;
 460
 461         if (!(ring->flags & KNAV_RING_FLAG_BUSY))
 462                 goto no_init;
 463
 464         k3_ringacc_ring_free_sci(ring);
 465
 466         dma_free_coherent(ringacc->dev,
 467                           ring->size * (4 << ring->elm_size),
 468                           ring->ring_mem_virt, ring->ring_mem_dma);
 469         ring->flags &= ~KNAV_RING_FLAG_BUSY;
 470         ring->ops = NULL;
 471         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 472                 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
 473                 ring->proxy = NULL;
 474                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
 475         }
 476
 477 no_init:
 478         clear_bit(ring->ring_id, ringacc->rings_inuse);
 479
 480         module_put(ringacc->dev->driver->owner);
 481
 482 out:
 483         return 0;
 484 }
 485
 486 u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
 487 {
 488         if (!ring)
 489                 return -EINVAL;
 490
 491         return ring->ring_id;
 492 }
 493
 494 static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
 495 {
 496         struct k3_nav_ringacc *ringacc = ring->parent;
 497         u32 ring_idx;
 498         int ret;
 499
 500         if (!ringacc->tisci)
 501                 return -EINVAL;
 502
 503         ring_idx = ring->ring_id;
 504         ret = ringacc->tisci_ring_ops->config(
 505                         ringacc->tisci,
 506                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 507                         ringacc->tisci_dev_id,
 508                         ring_idx,
 509                         lower_32_bits(ring->ring_mem_dma),
 510                         upper_32_bits(ring->ring_mem_dma),
 511                         ring->size,
 512                         ring->mode,
 513                         ring->elm_size,
 514                         0);
 515         if (ret)
 516                 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
 517                         ret, ring_idx);
 518
 519         return ret;
 520 }
 521
 522 int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
 523                             struct k3_nav_ring_cfg *cfg)
 524 {
 525         struct k3_nav_ringacc *ringacc = ring->parent;
 526         int ret = 0;
 527
 528         if (!ring || !cfg)
 529                 return -EINVAL;
 530         if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
 531             cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
 532             cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
 533             !test_bit(ring->ring_id, ringacc->rings_inuse))
 534                 return -EINVAL;
 535
 536         if (ring->use_count != 1)
 537                 return 0;
 538
 539         ring->size = cfg->size;
 540         ring->elm_size = cfg->elm_size;
 541         ring->mode = cfg->mode;
 542         ring->occ = 0;
 543         ring->free = 0;
 544         ring->rindex = 0;
 545         ring->windex = 0;
 546
 547         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
 548                 ring->proxy = ringacc->proxy_target_base +
 549                               ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
 550
 551         switch (ring->mode) {
 552         case K3_NAV_RINGACC_RING_MODE_RING:
 553                 ring->ops = &k3_nav_mode_ring_ops;
 554                 break;
 555         case K3_NAV_RINGACC_RING_MODE_QM:
 556                 /*
 557                  * In Queue mode elm_size can be 8 only and each operation
 558                  * uses 2 element slots
 559                  */
 560                 if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
 561                     cfg->size % 2)
 562                         goto err_free_proxy;
 563         case K3_NAV_RINGACC_RING_MODE_MESSAGE:
 564                 if (ring->proxy)
 565                         ring->ops = &k3_nav_mode_proxy_ops;
 566                 else
 567                         ring->ops = &k3_nav_mode_msg_ops;
 568                 break;
 569         default:
 570                 ring->ops = NULL;
 571                 ret = -EINVAL;
 572                 goto err_free_proxy;
 573         };
 574
 575         ring->ring_mem_virt =
 576                         dma_zalloc_coherent(ringacc->dev,
 577                                             ring->size * (4 << ring->elm_size),
 578                                             &ring->ring_mem_dma, GFP_KERNEL);
 579         if (!ring->ring_mem_virt) {
 580                 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
 581                 ret = -ENOMEM;
 582                 goto err_free_ops;
 583         }
 584
 585         ret = k3_nav_ringacc_ring_cfg_sci(ring);
 586
 587         if (ret)
 588                 goto err_free_mem;
 589
 590         ring->flags |= KNAV_RING_FLAG_BUSY;
 591         ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
 592                         K3_NAV_RING_FLAG_SHARED : 0;
 593
 594         return 0;
 595
 596 err_free_mem:
 597         dma_free_coherent(ringacc->dev,
 598                           ring->size * (4 << ring->elm_size),
 599                           ring->ring_mem_virt,
 600                           ring->ring_mem_dma);
 601 err_free_ops:
 602         ring->ops = NULL;
 603 err_free_proxy:
 604         ring->proxy = NULL;
 605         return ret;
 606 }
 607
 608 u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
 609 {
 610         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 611                 return -EINVAL;
 612
 613         return ring->size;
 614 }
 615
 616 u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
 617 {
 618         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 619                 return -EINVAL;
 620
 621         if (!ring->free)
 622                 ring->free = ring->size - ringacc_readl(&ring->rt->occ);
 623
 624         return ring->free;
 625 }
 626
 627 u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
 628 {
 629         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 630                 return -EINVAL;
 631
 632         return ringacc_readl(&ring->rt->occ);
 633 }
 634
 635 u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
 636 {
 637         return !k3_nav_ringacc_ring_get_free(ring);
 638 }
 639
 640 enum k3_ringacc_access_mode {
 641         K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
 642         K3_RINGACC_ACCESS_MODE_POP_HEAD,
 643         K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
 644         K3_RINGACC_ACCESS_MODE_POP_TAIL,
 645         K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
 646         K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
 647 };
 648
 649 static int k3_ringacc_ring_cfg_proxy(struct k3_nav_ring *ring,
 650                                      enum k3_ringacc_proxy_access_mode mode)
 651 {
 652         u32 val;
 653
 654         val = ring->ring_id;
 655         val |= mode << 16;
 656         val |= ring->elm_size << 24;
 657         ringacc_writel(val, &ring->proxy->control);
 658         return 0;
 659 }
 660
 661 static int k3_nav_ringacc_ring_access_proxy(
 662                         struct k3_nav_ring *ring, void *elem,
 663                         enum k3_ringacc_access_mode access_mode)
 664 {
 665         void __iomem *ptr;
 666
 667         ptr = (void __iomem *)&ring->proxy->data;
 668
 669         switch (access_mode) {
 670         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 671         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 672                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
 673                 break;
 674         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 675         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 676                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
 677                 break;
 678         default:
 679                 return -EINVAL;
 680         }
 681
 682         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 683
 684         switch (access_mode) {
 685         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 686         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 687                 pr_debug("proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 688                          ptr, access_mode);
 689                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 690                 ring->occ--;
 691                 break;
 692         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 693         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 694                 pr_debug("proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
 695                          ptr, access_mode);
 696                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 697                 ring->free--;
 698                 break;
 699         default:
 700                 return -EINVAL;
 701         }
 702
 703         pr_debug("proxy: free%d occ%d\n",
 704                  ring->free, ring->occ);
 705         return 0;
 706 }
 707
 708 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring, void *elem)
 709 {
 710         return k3_nav_ringacc_ring_access_proxy(
 711                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 712 }
 713
 714 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring, void *elem)
 715 {
 716         return k3_nav_ringacc_ring_access_proxy(
 717                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 718 }
 719
 720 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem)
 721 {
 722         return k3_nav_ringacc_ring_access_proxy(
 723                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 724 }
 725
 726 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem)
 727 {
 728         return k3_nav_ringacc_ring_access_proxy(
 729                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 730 }
 731
 732 static int k3_nav_ringacc_ring_access_io(
 733                 struct k3_nav_ring *ring, void *elem,
 734                 enum k3_ringacc_access_mode access_mode)
 735 {
 736         void __iomem *ptr;
 737
 738         switch (access_mode) {
 739         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 740         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 741                 ptr = (void __iomem *)&ring->fifos->head_data;
 742                 break;
 743         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 744         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 745                 ptr = (void __iomem *)&ring->fifos->tail_data;
 746                 break;
 747         default:
 748                 return -EINVAL;
 749         }
 750
 751         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 752
 753         switch (access_mode) {
 754         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 755         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 756                 pr_debug("memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 757                          ptr, access_mode);
 758                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 759                 ring->occ--;
 760                 break;
 761         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 762         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 763                 pr_debug("memcpy_toio(x): --> ptr(%p), mode:%d\n",
 764                          ptr, access_mode);
 765                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 766                 ring->free--;
 767                 break;
 768         default:
 769                 return -EINVAL;
 770         }
 771
 772         pr_debug("free%d index%d occ%d index%d\n",
 773                  ring->free, ring->windex, ring->occ, ring->rindex);
 774         return 0;
 775 }
 776
 777 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 778                                             void *elem)
 779 {
 780         return k3_nav_ringacc_ring_access_io(
 781                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 782 }
 783
 784 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem)
 785 {
 786         return k3_nav_ringacc_ring_access_io(
 787                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 788 }
 789
 790 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem)
 791 {
 792         return k3_nav_ringacc_ring_access_io(
 793                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 794 }
 795
 796 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring, void *elem)
 797 {
 798         return k3_nav_ringacc_ring_access_io(
 799                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 800 }
 801
 802 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
 803 {
 804         void *elem_ptr;
 805
 806         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->windex);
 807
 808         memcpy(elem_ptr, elem, (4 << ring->elm_size));
 809
 810         ring->windex = (ring->windex + 1) % ring->size;
 811         ring->free--;
 812         ringacc_writel(1, &ring->rt->db);
 813
 814         pr_debug("ring_push_mem: free%d index%d\n",
 815                  ring->free, ring->windex);
 816
 817         return 0;
 818 }
 819
 820 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
 821 {
 822         void *elem_ptr;
 823
 824         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->rindex);
 825
 826         memcpy(elem, elem_ptr, (4 << ring->elm_size));
 827
 828         ring->rindex = (ring->rindex + 1) % ring->size;
 829         ring->occ--;
 830         ringacc_writel(-1, &ring->rt->db);
 831
 832         pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
 833                  ring->occ, ring->rindex, elem_ptr);
 834         return 0;
 835 }
 836
 837 int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
 838 {
 839         int ret = -EOPNOTSUPP;
 840
 841         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 842                 return -EINVAL;
 843
 844         pr_debug("ring_push%d: free%d index%d\n",
 845                  ring->ring_id, ring->free, ring->windex);
 846
 847         if (k3_nav_ringacc_ring_is_full(ring))
 848                 return -ENOMEM;
 849
 850         if (ring->ops && ring->ops->push_tail)
 851                 ret = ring->ops->push_tail(ring, elem);
 852
 853         return ret;
 854 }
 855
 856 int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
 857 {
 858         int ret = -EOPNOTSUPP;
 859
 860         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 861                 return -EINVAL;
 862
 863         pr_debug("ring_push_head: free%d index%d\n",
 864                  ring->free, ring->windex);
 865
 866         if (k3_nav_ringacc_ring_is_full(ring))
 867                 return -ENOMEM;
 868
 869         if (ring->ops && ring->ops->push_head)
 870                 ret = ring->ops->push_head(ring, elem);
 871
 872         return ret;
 873 }
 874
 875 int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
 876 {
 877         int ret = -EOPNOTSUPP;
 878
 879         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 880                 return -EINVAL;
 881
 882         if (!ring->occ)
 883                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 884
 885         pr_debug("ring_pop%d: occ%d index%d\n",
 886                  ring->ring_id, ring->occ, ring->rindex);
 887
 888         if (!ring->occ)
 889                 return -ENODATA;
 890
 891         if (ring->ops && ring->ops->pop_head)
 892                 ret = ring->ops->pop_head(ring, elem);
 893
 894         return ret;
 895 }
 896
 897 int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
 898 {
 899         int ret = -EOPNOTSUPP;
 900
 901         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 902                 return -EINVAL;
 903
 904         if (!ring->occ)
 905                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 906
 907         pr_debug("ring_pop_tail: occ%d index%d\n",
 908                  ring->occ, ring->rindex);
 909
 910         if (!ring->occ)
 911                 return -ENODATA;
 912
 913         if (ring->ops && ring->ops->pop_tail)
 914                 ret = ring->ops->pop_tail(ring, elem);
 915
 916         return ret;
 917 }
 918
 919 static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
 920 {
 921         struct udevice *dev = ringacc->dev;
 922         struct udevice *tisci_dev = NULL;
 923         int ret;
 924
 925         ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
 926         if (!ringacc->num_rings) {
 927                 dev_err(dev, "ti,num-rings read failure %d\n", ret);
 928                 return -EINVAL;
 929         }
 930
 931         ringacc->dma_ring_reset_quirk =
 932                         dev_read_bool(dev, "ti,dma-ring-reset-quirk");
 933
 934         ret = uclass_get_device_by_name(UCLASS_FIRMWARE, "dmsc", &tisci_dev);
 935         if (ret) {
 936                 pr_debug("TISCI RA RM get failed (%d)\n", ret);
 937                 ringacc->tisci = NULL;
 938                 return -ENODEV;
 939         }
 940         ringacc->tisci = (struct ti_sci_handle *)
 941                          (ti_sci_get_handle_from_sysfw(tisci_dev));
 942
 943         ret = dev_read_u32_default(dev, "ti,sci", 0);
 944         if (!ret) {
 945                 dev_err(dev, "TISCI RA RM disabled\n");
 946                 ringacc->tisci = NULL;
 947                 return ret;
 948         }
 949
 950         ret = dev_read_u32(dev, "ti,sci-dev-id", &ringacc->tisci_dev_id);
 951         if (ret) {
 952                 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
 953                 ringacc->tisci = NULL;
 954                 return ret;
 955         }
 956
 957         ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
 958                                         ringacc->tisci, dev,
 959                                         ringacc->tisci_dev_id,
 960                                         "ti,sci-rm-range-gp-rings");
 961         if (IS_ERR(ringacc->rm_gp_range))
 962                 ret = PTR_ERR(ringacc->rm_gp_range);
 963
 964         return 0;
 965 }
 966
 967 static int k3_nav_ringacc_probe(struct udevice *dev)
 968 {
 969         struct k3_nav_ringacc *ringacc;
 970         void __iomem *base_fifo, *base_rt;
 971         int ret, i;
 972
 973         ringacc = dev_get_priv(dev);
 974         if (!ringacc)
 975                 return -ENOMEM;
 976
 977         ringacc->dev = dev;
 978
 979         ret = k3_nav_ringacc_probe_dt(ringacc);
 980         if (ret)
 981                 return ret;
 982
 983         base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
 984         pr_debug("rt %p\n", base_rt);
 985         if (IS_ERR(base_rt))
 986                 return PTR_ERR(base_rt);
 987
 988         base_fifo = (uint32_t *)devfdt_get_addr_name(dev, "fifos");
 989         pr_debug("fifos %p\n", base_fifo);
 990         if (IS_ERR(base_fifo))
 991                 return PTR_ERR(base_fifo);
 992
 993         ringacc->proxy_gcfg = (struct k3_ringacc_proxy_gcfg_regs __iomem *)
 994                 devfdt_get_addr_name(dev, "proxy_gcfg");
 995         if (IS_ERR(ringacc->proxy_gcfg))
 996                 return PTR_ERR(ringacc->proxy_gcfg);
 997         ringacc->proxy_target_base =
 998                 (struct k3_ringacc_proxy_gcfg_regs __iomem *)
 999                 devfdt_get_addr_name(dev, "proxy_target");
1000         if (IS_ERR(ringacc->proxy_target_base))
1001                 return PTR_ERR(ringacc->proxy_target_base);
1002
1003         ringacc->num_proxies = ringacc_readl(&ringacc->proxy_gcfg->config) &
1004                                          K3_RINGACC_PROXY_CFG_THREADS_MASK;
1005
1006         ringacc->rings = devm_kzalloc(dev,
1007                                       sizeof(*ringacc->rings) *
1008                                       ringacc->num_rings,
1009                                       GFP_KERNEL);
1010         ringacc->rings_inuse = devm_kcalloc(dev,
1011                                             BITS_TO_LONGS(ringacc->num_rings),
1012                                             sizeof(unsigned long), GFP_KERNEL);
1013         ringacc->proxy_inuse = devm_kcalloc(dev,
1014                                             BITS_TO_LONGS(ringacc->num_proxies),
1015                                             sizeof(unsigned long), GFP_KERNEL);
1016
1017         if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1018                 return -ENOMEM;
1019
1020         for (i = 0; i < ringacc->num_rings; i++) {
1021                 ringacc->rings[i].rt = base_rt +
1022                                        KNAV_RINGACC_RT_REGS_STEP * i;
1023                 ringacc->rings[i].fifos = base_fifo +
1024                                           KNAV_RINGACC_FIFO_REGS_STEP * i;
1025                 ringacc->rings[i].parent = ringacc;
1026                 ringacc->rings[i].ring_id = i;
1027                 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1028         }
1029         dev_set_drvdata(dev, ringacc);
1030
1031         ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1032
1033         list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
1034
1035         dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1036                  ringacc->num_rings,
1037                  ringacc->rm_gp_range->desc[0].start,
1038                  ringacc->rm_gp_range->desc[0].num,
1039                  ringacc->tisci_dev_id);
1040         dev_info(dev, "dma-ring-reset-quirk: %s\n",
1041                  ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1042         dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1043                  ringacc_readl(&ringacc->proxy_gcfg->revision),
1044                  ringacc->num_proxies);
1045         return 0;
1046 }
1047
1048 static const struct udevice_id knav_ringacc_ids[] = {
1049         { .compatible = "ti,am654-navss-ringacc" },
1050         {},
1051 };
1052
1053 U_BOOT_DRIVER(k3_navss_ringacc) = {
1054         .name   = "k3-navss-ringacc",
1055         .id     = UCLASS_MISC,
1056         .of_match = knav_ringacc_ids,
1057         .probe = k3_nav_ringacc_probe,
1058         .priv_auto_alloc_size = sizeof(struct k3_nav_ringacc),
1059 };