drivers/soc/ti/k3-navss-ringacc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
   4  *
   5  * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
   6  */
   7
   8 #include <common.h>
   9 #include <cpu_func.h>
  10 #include <asm/io.h>
  11 #include <malloc.h>
  12 #include <asm/dma-mapping.h>
  13 #include <asm/bitops.h>
  14 #include <dm.h>
  15 #include <dm/read.h>
  16 #include <dm/uclass.h>
  17 #include <linux/compat.h>
  18 #include <linux/soc/ti/k3-navss-ringacc.h>
  19 #include <linux/soc/ti/ti_sci_protocol.h>
  20
  21 #define set_bit(bit, bitmap)    __set_bit(bit, bitmap)
  22 #define clear_bit(bit, bitmap)  __clear_bit(bit, bitmap)
  23 #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
  24         dma_free_coherent(cpu_addr)
  25 #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
  26 ({ \
  27         void    *ring_mem_virt; \
  28         ring_mem_virt = dma_alloc_coherent((size), \
  29                                            (unsigned long *)(dma_handle)); \
  30         if (ring_mem_virt) \
  31                 memset(ring_mem_virt, 0, (size)); \
  32         ring_mem_virt; \
  33 })
  34
  35 static LIST_HEAD(k3_nav_ringacc_list);
  36
  37 static  void ringacc_writel(u32 v, void __iomem *reg)
  38 {
  39         pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
  40         writel(v, reg);
  41 }
  42
  43 static  u32 ringacc_readl(void __iomem *reg)
  44 {
  45         u32 v;
  46
  47         v = readl(reg);
  48         pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
  49         return v;
  50 }
  51
  52 #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK           GENMASK(19, 0)
  53
  54 /**
  55  * struct k3_nav_ring_rt_regs -  The RA Control/Status Registers region
  56  */
  57 struct k3_nav_ring_rt_regs {
  58         u32     resv_16[4];
  59         u32     db;             /* RT Ring N Doorbell Register */
  60         u32     resv_4[1];
  61         u32     occ;            /* RT Ring N Occupancy Register */
  62         u32     indx;           /* RT Ring N Current Index Register */
  63         u32     hwocc;          /* RT Ring N Hardware Occupancy Register */
  64         u32     hwindx;         /* RT Ring N Current Index Register */
  65 };
  66
  67 #define KNAV_RINGACC_RT_REGS_STEP       0x1000
  68
  69 /**
  70  * struct k3_nav_ring_fifo_regs -  The Ring Accelerator Queues Registers region
  71  */
  72 struct k3_nav_ring_fifo_regs {
  73         u32     head_data[128];         /* Ring Head Entry Data Registers */
  74         u32     tail_data[128];         /* Ring Tail Entry Data Registers */
  75         u32     peek_head_data[128];    /* Ring Peek Head Entry Data Regs */
  76         u32     peek_tail_data[128];    /* Ring Peek Tail Entry Data Regs */
  77 };
  78
  79 /**
  80  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
  81  */
  82 struct k3_ringacc_proxy_gcfg_regs {
  83         u32     revision;       /* Revision Register */
  84         u32     config;         /* Config Register */
  85 };
  86
  87 #define K3_RINGACC_PROXY_CFG_THREADS_MASK               GENMASK(15, 0)
  88
  89 /**
  90  * struct k3_ringacc_proxy_target_regs -  RA Proxy Datapath MMIO Region
  91  */
  92 struct k3_ringacc_proxy_target_regs {
  93         u32     control;        /* Proxy Control Register */
  94         u32     status;         /* Proxy Status Register */
  95         u8      resv_512[504];
  96         u32     data[128];      /* Proxy Data Register */
  97 };
  98
  99 #define K3_RINGACC_PROXY_TARGET_STEP    0x1000
 100 #define K3_RINGACC_PROXY_NOT_USED       (-1)
 101
 102 enum k3_ringacc_proxy_access_mode {
 103         PROXY_ACCESS_MODE_HEAD = 0,
 104         PROXY_ACCESS_MODE_TAIL = 1,
 105         PROXY_ACCESS_MODE_PEEK_HEAD = 2,
 106         PROXY_ACCESS_MODE_PEEK_TAIL = 3,
 107 };
 108
 109 #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
 110 #define KNAV_RINGACC_FIFO_REGS_STEP     0x1000
 111 #define KNAV_RINGACC_MAX_DB_RING_CNT    (127U)
 112
 113 /**
 114  * struct k3_nav_ring_ops -  Ring operations
 115  */
 116 struct k3_nav_ring_ops {
 117         int (*push_tail)(struct k3_nav_ring *ring, void *elm);
 118         int (*push_head)(struct k3_nav_ring *ring, void *elm);
 119         int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
 120         int (*pop_head)(struct k3_nav_ring *ring, void *elm);
 121 };
 122
 123 /**
 124  * struct k3_nav_ring - RA Ring descriptor
 125  *
 126  * @rt - Ring control/status registers
 127  * @fifos - Ring queues registers
 128  * @proxy - Ring Proxy Datapath registers
 129  * @ring_mem_dma - Ring buffer dma address
 130  * @ring_mem_virt - Ring buffer virt address
 131  * @ops - Ring operations
 132  * @size - Ring size in elements
 133  * @elm_size - Size of the ring element
 134  * @mode - Ring mode
 135  * @flags - flags
 136  * @free - Number of free elements
 137  * @occ - Ring occupancy
 138  * @windex - Write index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 139  * @rindex - Read index (only for @K3_NAV_RINGACC_RING_MODE_RING)
 140  * @ring_id - Ring Id
 141  * @parent - Pointer on struct @k3_nav_ringacc
 142  * @use_count - Use count for shared rings
 143  * @proxy_id - RA Ring Proxy Id (only if @K3_NAV_RINGACC_RING_USE_PROXY)
 144  */
 145 struct k3_nav_ring {
 146         struct k3_nav_ring_rt_regs __iomem *rt;
 147         struct k3_nav_ring_fifo_regs __iomem *fifos;
 148         struct k3_ringacc_proxy_target_regs  __iomem *proxy;
 149         dma_addr_t      ring_mem_dma;
 150         void            *ring_mem_virt;
 151         struct k3_nav_ring_ops *ops;
 152         u32             size;
 153         enum k3_nav_ring_size elm_size;
 154         enum k3_nav_ring_mode mode;
 155         u32             flags;
 156 #define KNAV_RING_FLAG_BUSY     BIT(1)
 157 #define K3_NAV_RING_FLAG_SHARED BIT(2)
 158         u32             free;
 159         u32             occ;
 160         u32             windex;
 161         u32             rindex;
 162         u32             ring_id;
 163         struct k3_nav_ringacc   *parent;
 164         u32             use_count;
 165         int             proxy_id;
 166 };
 167
 168 /**
 169  * struct k3_nav_ringacc - Rings accelerator descriptor
 170  *
 171  * @dev - pointer on RA device
 172  * @proxy_gcfg - RA proxy global config registers
 173  * @proxy_target_base - RA proxy datapath region
 174  * @num_rings - number of ring in RA
 175  * @rm_gp_range - general purpose rings range from tisci
 176  * @dma_ring_reset_quirk - DMA reset w/a enable
 177  * @num_proxies - number of RA proxies
 178  * @rings - array of rings descriptors (struct @k3_nav_ring)
 179  * @list - list of RAs in the system
 180  * @tisci - pointer ti-sci handle
 181  * @tisci_ring_ops - ti-sci rings ops
 182  * @tisci_dev_id - ti-sci device id
 183  */
 184 struct k3_nav_ringacc {
 185         struct udevice *dev;
 186         struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
 187         void __iomem *proxy_target_base;
 188         u32 num_rings; /* number of rings in Ringacc module */
 189         unsigned long *rings_inuse;
 190         struct ti_sci_resource *rm_gp_range;
 191         bool dma_ring_reset_quirk;
 192         u32 num_proxies;
 193         unsigned long *proxy_inuse;
 194
 195         struct k3_nav_ring *rings;
 196         struct list_head list;
 197
 198         const struct ti_sci_handle *tisci;
 199         const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
 200         u32  tisci_dev_id;
 201 };
 202
 203 static long k3_nav_ringacc_ring_get_fifo_pos(struct k3_nav_ring *ring)
 204 {
 205         return KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES -
 206                (4 << ring->elm_size);
 207 }
 208
 209 static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
 210 {
 211         return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
 212 }
 213
 214 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
 215 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
 216
 217 static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
 218                 .push_tail = k3_nav_ringacc_ring_push_mem,
 219                 .pop_head = k3_nav_ringacc_ring_pop_mem,
 220 };
 221
 222 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem);
 223 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem);
 224 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 225                                             void *elem);
 226 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring,
 227                                            void *elem);
 228
 229 static struct k3_nav_ring_ops k3_nav_mode_msg_ops = {
 230                 .push_tail = k3_nav_ringacc_ring_push_io,
 231                 .push_head = k3_nav_ringacc_ring_push_head_io,
 232                 .pop_tail = k3_nav_ringacc_ring_pop_tail_io,
 233                 .pop_head = k3_nav_ringacc_ring_pop_io,
 234 };
 235
 236 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring,
 237                                            void *elem);
 238 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring,
 239                                            void *elem);
 240 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem);
 241 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem);
 242
 243 static struct k3_nav_ring_ops k3_nav_mode_proxy_ops = {
 244                 .push_tail = k3_ringacc_ring_push_tail_proxy,
 245                 .push_head = k3_ringacc_ring_push_head_proxy,
 246                 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
 247                 .pop_head = k3_ringacc_ring_pop_head_proxy,
 248 };
 249
 250 struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
 251 {
 252         return ringacc->dev;
 253 }
 254
 255 struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
 256                                                 int id, u32 flags)
 257 {
 258         int proxy_id = K3_RINGACC_PROXY_NOT_USED;
 259
 260         if (id == K3_NAV_RINGACC_RING_ID_ANY) {
 261                 /* Request for any general purpose ring */
 262                 struct ti_sci_resource_desc *gp_rings =
 263                                         &ringacc->rm_gp_range->desc[0];
 264                 unsigned long size;
 265
 266                 size = gp_rings->start + gp_rings->num;
 267                 id = find_next_zero_bit(ringacc->rings_inuse,
 268                                         size, gp_rings->start);
 269                 if (id == size)
 270                         goto error;
 271         } else if (id < 0) {
 272                 goto error;
 273         }
 274
 275         if (test_bit(id, ringacc->rings_inuse) &&
 276             !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
 277                 goto error;
 278         else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
 279                 goto out;
 280
 281         if (flags & K3_NAV_RINGACC_RING_USE_PROXY) {
 282                 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
 283                                               ringacc->num_proxies, 0);
 284                 if (proxy_id == ringacc->num_proxies)
 285                         goto error;
 286         }
 287
 288         if (!try_module_get(ringacc->dev->driver->owner))
 289                 goto error;
 290
 291         if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 292                 set_bit(proxy_id, ringacc->proxy_inuse);
 293                 ringacc->rings[id].proxy_id = proxy_id;
 294                 pr_debug("Giving ring#%d proxy#%d\n",
 295                          id, proxy_id);
 296         } else {
 297                 pr_debug("Giving ring#%d\n", id);
 298         }
 299
 300         set_bit(id, ringacc->rings_inuse);
 301 out:
 302         ringacc->rings[id].use_count++;
 303         return &ringacc->rings[id];
 304
 305 error:
 306         return NULL;
 307 }
 308
 309 static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
 310 {
 311         struct k3_nav_ringacc *ringacc = ring->parent;
 312         int ret;
 313
 314         ret = ringacc->tisci_ring_ops->config(
 315                         ringacc->tisci,
 316                         TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
 317                         ringacc->tisci_dev_id,
 318                         ring->ring_id,
 319                         0,
 320                         0,
 321                         ring->size,
 322                         0,
 323                         0,
 324                         0);
 325         if (ret)
 326                 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
 327                         ret, ring->ring_id);
 328 }
 329
 330 void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
 331 {
 332         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 333                 return;
 334
 335         ring->occ = 0;
 336         ring->free = 0;
 337         ring->rindex = 0;
 338         ring->windex = 0;
 339
 340         k3_ringacc_ring_reset_sci(ring);
 341 }
 342
 343 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
 344                                                enum k3_nav_ring_mode mode)
 345 {
 346         struct k3_nav_ringacc *ringacc = ring->parent;
 347         int ret;
 348
 349         ret = ringacc->tisci_ring_ops->config(
 350                         ringacc->tisci,
 351                         TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
 352                         ringacc->tisci_dev_id,
 353                         ring->ring_id,
 354                         0,
 355                         0,
 356                         0,
 357                         mode,
 358                         0,
 359                         0);
 360         if (ret)
 361                 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
 362                         ret, ring->ring_id);
 363 }
 364
 365 void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
 366 {
 367         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 368                 return;
 369
 370         if (!ring->parent->dma_ring_reset_quirk) {
 371                 k3_nav_ringacc_ring_reset(ring);
 372                 return;
 373         }
 374
 375         if (!occ)
 376                 occ = ringacc_readl(&ring->rt->occ);
 377
 378         if (occ) {
 379                 u32 db_ring_cnt, db_ring_cnt_cur;
 380
 381                 pr_debug("%s %u occ: %u\n", __func__,
 382                          ring->ring_id, occ);
 383                 /* 2. Reset the ring */
 384                 k3_ringacc_ring_reset_sci(ring);
 385
 386                 /*
 387                  * 3. Setup the ring in ring/doorbell mode
 388                  * (if not already in this mode)
 389                  */
 390                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 391                         k3_ringacc_ring_reconfig_qmode_sci(
 392                                         ring, K3_NAV_RINGACC_RING_MODE_RING);
 393                 /*
 394                  * 4. Ring the doorbell 2**22 – ringOcc times.
 395                  * This will wrap the internal UDMAP ring state occupancy
 396                  * counter (which is 21-bits wide) to 0.
 397                  */
 398                 db_ring_cnt = (1U << 22) - occ;
 399
 400                 while (db_ring_cnt != 0) {
 401                         /*
 402                          * Ring the doorbell with the maximum count each
 403                          * iteration if possible to minimize the total
 404                          * of writes
 405                          */
 406                         if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
 407                                 db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
 408                         else
 409                                 db_ring_cnt_cur = db_ring_cnt;
 410
 411                         writel(db_ring_cnt_cur, &ring->rt->db);
 412                         db_ring_cnt -= db_ring_cnt_cur;
 413                 }
 414
 415                 /* 5. Restore the original ring mode (if not ring mode) */
 416                 if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
 417                         k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
 418         }
 419
 420         /* 2. Reset the ring */
 421         k3_nav_ringacc_ring_reset(ring);
 422 }
 423
 424 static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
 425 {
 426         struct k3_nav_ringacc *ringacc = ring->parent;
 427         int ret;
 428
 429         ret = ringacc->tisci_ring_ops->config(
 430                         ringacc->tisci,
 431                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 432                         ringacc->tisci_dev_id,
 433                         ring->ring_id,
 434                         0,
 435                         0,
 436                         0,
 437                         0,
 438                         0,
 439                         0);
 440         if (ret)
 441                 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
 442                         ret, ring->ring_id);
 443 }
 444
 445 int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
 446 {
 447         struct k3_nav_ringacc *ringacc;
 448
 449         if (!ring)
 450                 return -EINVAL;
 451
 452         ringacc = ring->parent;
 453
 454         pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
 455
 456         if (!test_bit(ring->ring_id, ringacc->rings_inuse))
 457                 return -EINVAL;
 458
 459         if (--ring->use_count)
 460                 goto out;
 461
 462         if (!(ring->flags & KNAV_RING_FLAG_BUSY))
 463                 goto no_init;
 464
 465         k3_ringacc_ring_free_sci(ring);
 466
 467         dma_free_coherent(ringacc->dev,
 468                           ring->size * (4 << ring->elm_size),
 469                           ring->ring_mem_virt, ring->ring_mem_dma);
 470         ring->flags &= ~KNAV_RING_FLAG_BUSY;
 471         ring->ops = NULL;
 472         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 473                 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
 474                 ring->proxy = NULL;
 475                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
 476         }
 477
 478 no_init:
 479         clear_bit(ring->ring_id, ringacc->rings_inuse);
 480
 481         module_put(ringacc->dev->driver->owner);
 482
 483 out:
 484         return 0;
 485 }
 486
 487 u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
 488 {
 489         if (!ring)
 490                 return -EINVAL;
 491
 492         return ring->ring_id;
 493 }
 494
 495 static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
 496 {
 497         struct k3_nav_ringacc *ringacc = ring->parent;
 498         u32 ring_idx;
 499         int ret;
 500
 501         if (!ringacc->tisci)
 502                 return -EINVAL;
 503
 504         ring_idx = ring->ring_id;
 505         ret = ringacc->tisci_ring_ops->config(
 506                         ringacc->tisci,
 507                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 508                         ringacc->tisci_dev_id,
 509                         ring_idx,
 510                         lower_32_bits(ring->ring_mem_dma),
 511                         upper_32_bits(ring->ring_mem_dma),
 512                         ring->size,
 513                         ring->mode,
 514                         ring->elm_size,
 515                         0);
 516         if (ret)
 517                 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
 518                         ret, ring_idx);
 519
 520         return ret;
 521 }
 522
 523 int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
 524                             struct k3_nav_ring_cfg *cfg)
 525 {
 526         struct k3_nav_ringacc *ringacc = ring->parent;
 527         int ret = 0;
 528
 529         if (!ring || !cfg)
 530                 return -EINVAL;
 531         if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
 532             cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
 533             cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
 534             !test_bit(ring->ring_id, ringacc->rings_inuse))
 535                 return -EINVAL;
 536
 537         if (ring->use_count != 1)
 538                 return 0;
 539
 540         ring->size = cfg->size;
 541         ring->elm_size = cfg->elm_size;
 542         ring->mode = cfg->mode;
 543         ring->occ = 0;
 544         ring->free = 0;
 545         ring->rindex = 0;
 546         ring->windex = 0;
 547
 548         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
 549                 ring->proxy = ringacc->proxy_target_base +
 550                               ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
 551
 552         switch (ring->mode) {
 553         case K3_NAV_RINGACC_RING_MODE_RING:
 554                 ring->ops = &k3_nav_mode_ring_ops;
 555                 break;
 556         case K3_NAV_RINGACC_RING_MODE_QM:
 557                 /*
 558                  * In Queue mode elm_size can be 8 only and each operation
 559                  * uses 2 element slots
 560                  */
 561                 if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
 562                     cfg->size % 2)
 563                         goto err_free_proxy;
 564         case K3_NAV_RINGACC_RING_MODE_MESSAGE:
 565                 if (ring->proxy)
 566                         ring->ops = &k3_nav_mode_proxy_ops;
 567                 else
 568                         ring->ops = &k3_nav_mode_msg_ops;
 569                 break;
 570         default:
 571                 ring->ops = NULL;
 572                 ret = -EINVAL;
 573                 goto err_free_proxy;
 574         };
 575
 576         ring->ring_mem_virt =
 577                         dma_zalloc_coherent(ringacc->dev,
 578                                             ring->size * (4 << ring->elm_size),
 579                                             &ring->ring_mem_dma, GFP_KERNEL);
 580         if (!ring->ring_mem_virt) {
 581                 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
 582                 ret = -ENOMEM;
 583                 goto err_free_ops;
 584         }
 585
 586         ret = k3_nav_ringacc_ring_cfg_sci(ring);
 587
 588         if (ret)
 589                 goto err_free_mem;
 590
 591         ring->flags |= KNAV_RING_FLAG_BUSY;
 592         ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
 593                         K3_NAV_RING_FLAG_SHARED : 0;
 594
 595         return 0;
 596
 597 err_free_mem:
 598         dma_free_coherent(ringacc->dev,
 599                           ring->size * (4 << ring->elm_size),
 600                           ring->ring_mem_virt,
 601                           ring->ring_mem_dma);
 602 err_free_ops:
 603         ring->ops = NULL;
 604 err_free_proxy:
 605         ring->proxy = NULL;
 606         return ret;
 607 }
 608
 609 u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
 610 {
 611         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 612                 return -EINVAL;
 613
 614         return ring->size;
 615 }
 616
 617 u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
 618 {
 619         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 620                 return -EINVAL;
 621
 622         if (!ring->free)
 623                 ring->free = ring->size - ringacc_readl(&ring->rt->occ);
 624
 625         return ring->free;
 626 }
 627
 628 u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
 629 {
 630         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 631                 return -EINVAL;
 632
 633         return ringacc_readl(&ring->rt->occ);
 634 }
 635
 636 u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
 637 {
 638         return !k3_nav_ringacc_ring_get_free(ring);
 639 }
 640
 641 enum k3_ringacc_access_mode {
 642         K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
 643         K3_RINGACC_ACCESS_MODE_POP_HEAD,
 644         K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
 645         K3_RINGACC_ACCESS_MODE_POP_TAIL,
 646         K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
 647         K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
 648 };
 649
 650 static int k3_ringacc_ring_cfg_proxy(struct k3_nav_ring *ring,
 651                                      enum k3_ringacc_proxy_access_mode mode)
 652 {
 653         u32 val;
 654
 655         val = ring->ring_id;
 656         val |= mode << 16;
 657         val |= ring->elm_size << 24;
 658         ringacc_writel(val, &ring->proxy->control);
 659         return 0;
 660 }
 661
 662 static int k3_nav_ringacc_ring_access_proxy(
 663                         struct k3_nav_ring *ring, void *elem,
 664                         enum k3_ringacc_access_mode access_mode)
 665 {
 666         void __iomem *ptr;
 667
 668         ptr = (void __iomem *)&ring->proxy->data;
 669
 670         switch (access_mode) {
 671         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 672         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 673                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
 674                 break;
 675         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 676         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 677                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
 678                 break;
 679         default:
 680                 return -EINVAL;
 681         }
 682
 683         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 684
 685         switch (access_mode) {
 686         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 687         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 688                 pr_debug("proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 689                          ptr, access_mode);
 690                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 691                 ring->occ--;
 692                 break;
 693         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 694         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 695                 pr_debug("proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
 696                          ptr, access_mode);
 697                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 698                 ring->free--;
 699                 break;
 700         default:
 701                 return -EINVAL;
 702         }
 703
 704         pr_debug("proxy: free%d occ%d\n",
 705                  ring->free, ring->occ);
 706         return 0;
 707 }
 708
 709 static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring, void *elem)
 710 {
 711         return k3_nav_ringacc_ring_access_proxy(
 712                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 713 }
 714
 715 static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring, void *elem)
 716 {
 717         return k3_nav_ringacc_ring_access_proxy(
 718                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 719 }
 720
 721 static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem)
 722 {
 723         return k3_nav_ringacc_ring_access_proxy(
 724                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 725 }
 726
 727 static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem)
 728 {
 729         return k3_nav_ringacc_ring_access_proxy(
 730                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 731 }
 732
 733 static int k3_nav_ringacc_ring_access_io(
 734                 struct k3_nav_ring *ring, void *elem,
 735                 enum k3_ringacc_access_mode access_mode)
 736 {
 737         void __iomem *ptr;
 738
 739         switch (access_mode) {
 740         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 741         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 742                 ptr = (void __iomem *)&ring->fifos->head_data;
 743                 break;
 744         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 745         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 746                 ptr = (void __iomem *)&ring->fifos->tail_data;
 747                 break;
 748         default:
 749                 return -EINVAL;
 750         }
 751
 752         ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
 753
 754         switch (access_mode) {
 755         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 756         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 757                 pr_debug("memcpy_fromio(x): --> ptr(%p), mode:%d\n",
 758                          ptr, access_mode);
 759                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 760                 ring->occ--;
 761                 break;
 762         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 763         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 764                 pr_debug("memcpy_toio(x): --> ptr(%p), mode:%d\n",
 765                          ptr, access_mode);
 766                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 767                 ring->free--;
 768                 break;
 769         default:
 770                 return -EINVAL;
 771         }
 772
 773         pr_debug("free%d index%d occ%d index%d\n",
 774                  ring->free, ring->windex, ring->occ, ring->rindex);
 775         return 0;
 776 }
 777
 778 static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
 779                                             void *elem)
 780 {
 781         return k3_nav_ringacc_ring_access_io(
 782                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 783 }
 784
 785 static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem)
 786 {
 787         return k3_nav_ringacc_ring_access_io(
 788                         ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 789 }
 790
 791 static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem)
 792 {
 793         return k3_nav_ringacc_ring_access_io(
 794                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 795 }
 796
 797 static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring, void *elem)
 798 {
 799         return k3_nav_ringacc_ring_access_io(
 800                         ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
 801 }
 802
 803 static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
 804 {
 805         void *elem_ptr;
 806
 807         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->windex);
 808
 809         memcpy(elem_ptr, elem, (4 << ring->elm_size));
 810
 811         flush_dcache_range((unsigned long)ring->ring_mem_virt,
 812                            ALIGN((unsigned long)ring->ring_mem_virt +
 813                                  ring->size * (4 << ring->elm_size),
 814                                  ARCH_DMA_MINALIGN));
 815
 816         ring->windex = (ring->windex + 1) % ring->size;
 817         ring->free--;
 818         ringacc_writel(1, &ring->rt->db);
 819
 820         pr_debug("ring_push_mem: free%d index%d\n",
 821                  ring->free, ring->windex);
 822
 823         return 0;
 824 }
 825
 826 static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
 827 {
 828         void *elem_ptr;
 829
 830         elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->rindex);
 831
 832         invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
 833                                 ALIGN((unsigned long)ring->ring_mem_virt +
 834                                       ring->size * (4 << ring->elm_size),
 835                                       ARCH_DMA_MINALIGN));
 836
 837         memcpy(elem, elem_ptr, (4 << ring->elm_size));
 838
 839         ring->rindex = (ring->rindex + 1) % ring->size;
 840         ring->occ--;
 841         ringacc_writel(-1, &ring->rt->db);
 842
 843         pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
 844                  ring->occ, ring->rindex, elem_ptr);
 845         return 0;
 846 }
 847
 848 int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
 849 {
 850         int ret = -EOPNOTSUPP;
 851
 852         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 853                 return -EINVAL;
 854
 855         pr_debug("ring_push%d: free%d index%d\n",
 856                  ring->ring_id, ring->free, ring->windex);
 857
 858         if (k3_nav_ringacc_ring_is_full(ring))
 859                 return -ENOMEM;
 860
 861         if (ring->ops && ring->ops->push_tail)
 862                 ret = ring->ops->push_tail(ring, elem);
 863
 864         return ret;
 865 }
 866
 867 int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
 868 {
 869         int ret = -EOPNOTSUPP;
 870
 871         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 872                 return -EINVAL;
 873
 874         pr_debug("ring_push_head: free%d index%d\n",
 875                  ring->free, ring->windex);
 876
 877         if (k3_nav_ringacc_ring_is_full(ring))
 878                 return -ENOMEM;
 879
 880         if (ring->ops && ring->ops->push_head)
 881                 ret = ring->ops->push_head(ring, elem);
 882
 883         return ret;
 884 }
 885
 886 int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
 887 {
 888         int ret = -EOPNOTSUPP;
 889
 890         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 891                 return -EINVAL;
 892
 893         if (!ring->occ)
 894                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 895
 896         pr_debug("ring_pop%d: occ%d index%d\n",
 897                  ring->ring_id, ring->occ, ring->rindex);
 898
 899         if (!ring->occ)
 900                 return -ENODATA;
 901
 902         if (ring->ops && ring->ops->pop_head)
 903                 ret = ring->ops->pop_head(ring, elem);
 904
 905         return ret;
 906 }
 907
 908 int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
 909 {
 910         int ret = -EOPNOTSUPP;
 911
 912         if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
 913                 return -EINVAL;
 914
 915         if (!ring->occ)
 916                 ring->occ = k3_nav_ringacc_ring_get_occ(ring);
 917
 918         pr_debug("ring_pop_tail: occ%d index%d\n",
 919                  ring->occ, ring->rindex);
 920
 921         if (!ring->occ)
 922                 return -ENODATA;
 923
 924         if (ring->ops && ring->ops->pop_tail)
 925                 ret = ring->ops->pop_tail(ring, elem);
 926
 927         return ret;
 928 }
 929
 930 static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
 931 {
 932         struct udevice *dev = ringacc->dev;
 933         struct udevice *tisci_dev = NULL;
 934         int ret;
 935
 936         ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
 937         if (!ringacc->num_rings) {
 938                 dev_err(dev, "ti,num-rings read failure %d\n", ret);
 939                 return -EINVAL;
 940         }
 941
 942         ringacc->dma_ring_reset_quirk =
 943                         dev_read_bool(dev, "ti,dma-ring-reset-quirk");
 944
 945         ret = uclass_get_device_by_name(UCLASS_FIRMWARE, "dmsc", &tisci_dev);
 946         if (ret) {
 947                 pr_debug("TISCI RA RM get failed (%d)\n", ret);
 948                 ringacc->tisci = NULL;
 949                 return -ENODEV;
 950         }
 951         ringacc->tisci = (struct ti_sci_handle *)
 952                          (ti_sci_get_handle_from_sysfw(tisci_dev));
 953
 954         ret = dev_read_u32_default(dev, "ti,sci", 0);
 955         if (!ret) {
 956                 dev_err(dev, "TISCI RA RM disabled\n");
 957                 ringacc->tisci = NULL;
 958                 return ret;
 959         }
 960
 961         ret = dev_read_u32(dev, "ti,sci-dev-id", &ringacc->tisci_dev_id);
 962         if (ret) {
 963                 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
 964                 ringacc->tisci = NULL;
 965                 return ret;
 966         }
 967
 968         ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
 969                                         ringacc->tisci, dev,
 970                                         ringacc->tisci_dev_id,
 971                                         "ti,sci-rm-range-gp-rings");
 972         if (IS_ERR(ringacc->rm_gp_range))
 973                 ret = PTR_ERR(ringacc->rm_gp_range);
 974
 975         return 0;
 976 }
 977
 978 static int k3_nav_ringacc_probe(struct udevice *dev)
 979 {
 980         struct k3_nav_ringacc *ringacc;
 981         void __iomem *base_fifo, *base_rt;
 982         int ret, i;
 983
 984         ringacc = dev_get_priv(dev);
 985         if (!ringacc)
 986                 return -ENOMEM;
 987
 988         ringacc->dev = dev;
 989
 990         ret = k3_nav_ringacc_probe_dt(ringacc);
 991         if (ret)
 992                 return ret;
 993
 994         base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
 995         pr_debug("rt %p\n", base_rt);
 996         if (IS_ERR(base_rt))
 997                 return PTR_ERR(base_rt);
 998
 999         base_fifo = (uint32_t *)devfdt_get_addr_name(dev, "fifos");
1000         pr_debug("fifos %p\n", base_fifo);
1001         if (IS_ERR(base_fifo))
1002                 return PTR_ERR(base_fifo);
1003
1004         ringacc->proxy_gcfg = (struct k3_ringacc_proxy_gcfg_regs __iomem *)
1005                 devfdt_get_addr_name(dev, "proxy_gcfg");
1006         if (IS_ERR(ringacc->proxy_gcfg))
1007                 return PTR_ERR(ringacc->proxy_gcfg);
1008         ringacc->proxy_target_base =
1009                 (struct k3_ringacc_proxy_gcfg_regs __iomem *)
1010                 devfdt_get_addr_name(dev, "proxy_target");
1011         if (IS_ERR(ringacc->proxy_target_base))
1012                 return PTR_ERR(ringacc->proxy_target_base);
1013
1014         ringacc->num_proxies = ringacc_readl(&ringacc->proxy_gcfg->config) &
1015                                          K3_RINGACC_PROXY_CFG_THREADS_MASK;
1016
1017         ringacc->rings = devm_kzalloc(dev,
1018                                       sizeof(*ringacc->rings) *
1019                                       ringacc->num_rings,
1020                                       GFP_KERNEL);
1021         ringacc->rings_inuse = devm_kcalloc(dev,
1022                                             BITS_TO_LONGS(ringacc->num_rings),
1023                                             sizeof(unsigned long), GFP_KERNEL);
1024         ringacc->proxy_inuse = devm_kcalloc(dev,
1025                                             BITS_TO_LONGS(ringacc->num_proxies),
1026                                             sizeof(unsigned long), GFP_KERNEL);
1027
1028         if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1029                 return -ENOMEM;
1030
1031         for (i = 0; i < ringacc->num_rings; i++) {
1032                 ringacc->rings[i].rt = base_rt +
1033                                        KNAV_RINGACC_RT_REGS_STEP * i;
1034                 ringacc->rings[i].fifos = base_fifo +
1035                                           KNAV_RINGACC_FIFO_REGS_STEP * i;
1036                 ringacc->rings[i].parent = ringacc;
1037                 ringacc->rings[i].ring_id = i;
1038                 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1039         }
1040         dev_set_drvdata(dev, ringacc);
1041
1042         ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1043
1044         list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
1045
1046         dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1047                  ringacc->num_rings,
1048                  ringacc->rm_gp_range->desc[0].start,
1049                  ringacc->rm_gp_range->desc[0].num,
1050                  ringacc->tisci_dev_id);
1051         dev_info(dev, "dma-ring-reset-quirk: %s\n",
1052                  ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1053         dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1054                  ringacc_readl(&ringacc->proxy_gcfg->revision),
1055                  ringacc->num_proxies);
1056         return 0;
1057 }
1058
1059 static const struct udevice_id knav_ringacc_ids[] = {
1060         { .compatible = "ti,am654-navss-ringacc" },
1061         {},
1062 };
1063
1064 U_BOOT_DRIVER(k3_navss_ringacc) = {
1065         .name   = "k3-navss-ringacc",
1066         .id     = UCLASS_MISC,
1067         .of_match = knav_ringacc_ids,
1068         .probe = k3_nav_ringacc_probe,
1069         .priv_auto_alloc_size = sizeof(struct k3_nav_ringacc),
1070 };